PVE/VZDump/QemuServer.pm

   1 package PVE::VZDump::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Basename;
   7 use File::Path;
   8 use IO::File;
   9 use IPC::Open3;
  10 use JSON;
  11 use POSIX qw(EINTR EAGAIN);
  12
  13 use PVE::Cluster qw(cfs_read_file);
  14 use PVE::INotify;
  15 use PVE::IPCC;
  16 use PVE::JSONSchema;
  17 use PVE::PBSClient;
  18 use PVE::QMPClient;
  19 use PVE::Storage::Plugin;
  20 use PVE::Storage::PBSPlugin;
  21 use PVE::Storage;
  22 use PVE::Tools;
  23 use PVE::VZDump;
  24 use PVE::Format qw(render_duration render_bytes);
  25
  26 use PVE::QemuConfig;
  27 use PVE::QemuServer;
  28 use PVE::QemuServer::Machine;
  29 use PVE::QemuServer::Monitor qw(mon_cmd);
  30
  31 use base qw (PVE::VZDump::Plugin);
  32
  33 sub new {
  34     my ($class, $vzdump) = @_;
  35
  36     PVE::VZDump::check_bin('qm');
  37
  38     my $self = bless { vzdump => $vzdump }, $class;
  39
  40     $self->{vmlist} = PVE::QemuServer::vzlist();
  41     $self->{storecfg} = PVE::Storage::config();
  42
  43     return $self;
  44 };
  45
  46 sub type {
  47     return 'qemu';
  48 }
  49
  50 sub vmlist {
  51     my ($self) = @_;
  52     return [ keys %{$self->{vmlist}} ];
  53 }
  54
  55 sub prepare {
  56     my ($self, $task, $vmid, $mode) = @_;
  57
  58     $task->{disks} = [];
  59
  60     my $conf = $self->{vmlist}->{$vmid} = PVE::QemuConfig->load_config($vmid);
  61
  62     $self->loginfo("VM Name: $conf->{name}")
  63         if defined($conf->{name});
  64
  65     $self->{vm_was_running} = 1;
  66     $self->{vm_was_paused} = 0;
  67     if (!PVE::QemuServer::check_running($vmid)) {
  68         $self->{vm_was_running} = 0;
  69     } elsif (PVE::QemuServer::vm_is_paused($vmid)) {
  70         $self->{vm_was_paused} = 1;
  71     }
  72
  73     $task->{hostname} = $conf->{name};
  74
  75     my $hostname = PVE::INotify::nodename();
  76
  77     my $vollist = [];
  78     my $drivehash = {};
  79     my $backup_volumes = PVE::QemuConfig->get_backup_volumes($conf);
  80
  81     foreach my $volume (@{$backup_volumes}) {
  82         my $name = $volume->{key};
  83         my $volume_config = $volume->{volume_config};
  84         my $volid = $volume_config->{file};
  85
  86         if (!$volume->{included}) {
  87             $self->loginfo("exclude disk '$name' '$volid' ($volume->{reason})");
  88             next;
  89         } elsif ($self->{vm_was_running} && $volume_config->{iothread}) {
  90             if (!PVE::QemuServer::Machine::runs_at_least_qemu_version($vmid, 4, 0, 1)) {
  91                 die "disk '$name' '$volid' (iothread=on) can't use backup feature with running QEMU " .
  92                     "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n";
  93             }
  94         } else {
  95             my $log = "include disk '$name' '$volid'";
  96             if (defined(my $size = $volume_config->{size})) {
  97                 my $readable_size = PVE::JSONSchema::format_size($size);
  98                 $log .= " $readable_size";
  99             }
 100             $self->loginfo($log);
 101         }
 102
 103         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 104         push @$vollist, $volid if $storeid;
 105         $drivehash->{$name} = $volume->{volume_config};
 106     }
 107
 108     PVE::Storage::activate_volumes($self->{storecfg}, $vollist);
 109
 110     foreach my $ds (sort keys %$drivehash) {
 111         my $drive = $drivehash->{$ds};
 112
 113         my $volid = $drive->{file};
 114         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 115
 116         my $path = $volid;
 117         if ($storeid) {
 118             $path = PVE::Storage::path($self->{storecfg}, $volid);
 119         }
 120         next if !$path;
 121
 122         my ($size, $format) = eval { PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5) };
 123         die "no such volume '$volid'\n" if $@;
 124
 125         my $diskinfo = {
 126             path => $path,
 127             volid => $volid,
 128             storeid => $storeid,
 129             format => $format,
 130             virtdev => $ds,
 131             qmdevice => "drive-$ds",
 132         };
 133
 134         if (-b $path) {
 135             $diskinfo->{type} = 'block';
 136         } else {
 137             $diskinfo->{type} = 'file';
 138         }
 139
 140         push @{$task->{disks}}, $diskinfo;
 141     }
 142 }
 143
 144 sub vm_status {
 145     my ($self, $vmid) = @_;
 146
 147     my $running = PVE::QemuServer::check_running($vmid) ? 1 : 0;
 148
 149     return wantarray ? ($running, $running ? 'running' : 'stopped') : $running;
 150 }
 151
 152 sub lock_vm {
 153     my ($self, $vmid) = @_;
 154
 155     PVE::QemuConfig->set_lock($vmid, 'backup');
 156 }
 157
 158 sub unlock_vm {
 159     my ($self, $vmid) = @_;
 160
 161     PVE::QemuConfig->remove_lock($vmid, 'backup');
 162 }
 163
 164 sub stop_vm {
 165     my ($self, $task, $vmid) = @_;
 166
 167     my $opts = $self->{vzdump}->{opts};
 168
 169     my $wait = $opts->{stopwait} * 60;
 170     # send shutdown and wait
 171     $self->cmd ("qm shutdown $vmid --skiplock --keepActive --timeout $wait");
 172 }
 173
 174 sub start_vm {
 175     my ($self, $task, $vmid) = @_;
 176
 177     $self->cmd ("qm start $vmid --skiplock");
 178 }
 179
 180 sub suspend_vm {
 181     my ($self, $task, $vmid) = @_;
 182
 183     return if $self->{vm_was_paused};
 184
 185     $self->cmd ("qm suspend $vmid --skiplock");
 186 }
 187
 188 sub resume_vm {
 189     my ($self, $task, $vmid) = @_;
 190
 191     return if $self->{vm_was_paused};
 192
 193     $self->cmd ("qm resume $vmid --skiplock");
 194 }
 195
 196 sub assemble {
 197     my ($self, $task, $vmid) = @_;
 198
 199     my $conffile = PVE::QemuConfig->config_file($vmid);
 200
 201     my $outfile = "$task->{tmpdir}/qemu-server.conf";
 202     my $firewall_src = "/etc/pve/firewall/$vmid.fw";
 203     my $firewall_dest = "$task->{tmpdir}/qemu-server.fw";
 204
 205     my $outfd = IO::File->new (">$outfile") ||
 206         die "unable to open '$outfile'";
 207     my $conffd = IO::File->new ($conffile, 'r') ||
 208         die "unable open '$conffile'";
 209
 210     my $found_snapshot;
 211     my $found_pending;
 212     while (defined (my $line = <$conffd>)) {
 213         next if $line =~ m/^\#vzdump\#/; # just to be sure
 214         next if $line =~ m/^\#qmdump\#/; # just to be sure
 215         if ($line =~ m/^\[(.*)\]\s*$/) {
 216             if ($1 =~ m/PENDING/i) {
 217                 $found_pending = 1;
 218             } else {
 219                 $found_snapshot = 1;
 220             }
 221         }
 222         next if $found_snapshot || $found_pending; # skip all snapshots and pending changes config data
 223
 224         if ($line =~ m/^unused\d+:\s*(\S+)\s*/) {
 225             $self->loginfo("skip unused drive '$1' (not included into backup)");
 226             next;
 227         }
 228         next if $line =~ m/^lock:/ || $line =~ m/^parent:/;
 229
 230         print $outfd $line;
 231     }
 232
 233     foreach my $di (@{$task->{disks}}) {
 234         if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 235             my $storeid = $di->{storeid} || '';
 236             my $format = $di->{format} || '';
 237             print $outfd "#qmdump#map:$di->{virtdev}:$di->{qmdevice}:$storeid:$format:\n";
 238         } else {
 239             die "internal error";
 240         }
 241     }
 242
 243     if ($found_snapshot) {
 244         $self->loginfo("snapshots found (not included into backup)");
 245     }
 246     if ($found_pending) {
 247         $self->loginfo("pending configuration changes found (not included into backup)");
 248     }
 249
 250     PVE::Tools::file_copy($firewall_src, $firewall_dest) if -f $firewall_src;
 251 }
 252
 253 sub archive {
 254     my ($self, $task, $vmid, $filename, $comp) = @_;
 255
 256     my $opts = $self->{vzdump}->{opts};
 257     my $scfg = $opts->{scfg};
 258
 259     if ($self->{vzdump}->{opts}->{pbs}) {
 260         $self->archive_pbs($task, $vmid);
 261     } else {
 262         $self->archive_vma($task, $vmid, $filename, $comp);
 263     }
 264 }
 265
 266 my $bitmap_action_to_human = sub {
 267     my ($self, $info) = @_;
 268
 269     my $action = $info->{action};
 270
 271     if ($action eq "not-used") {
 272         return "disabled (no support)";
 273     } elsif ($action eq "not-used-removed") {
 274         return "disabled (old bitmap cleared)";
 275     } elsif ($action eq "new") {
 276         return "created new";
 277     } elsif ($action eq "used") {
 278         if ($info->{dirty} == 0) {
 279             return "OK (drive clean)";
 280         } else {
 281             my $size = render_bytes($info->{size}, 1);
 282             my $dirty = render_bytes($info->{dirty}, 1);
 283             return "OK ($dirty of $size dirty)";
 284         }
 285     } elsif ($action eq "invalid") {
 286         return "existing bitmap was invalid and has been cleared";
 287     } else {
 288         return "unknown";
 289     }
 290 };
 291
 292 my $query_backup_status_loop = sub {
 293     my ($self, $vmid, $job_uuid, $qemu_support) = @_;
 294
 295     my $starttime = time ();
 296     my $last_time = $starttime;
 297     my ($last_percent, $last_total, $last_target, $last_zero, $last_transferred) = (-1, 0, 0, 0, 0);
 298     my ($transferred, $reused);
 299
 300     my $get_mbps = sub {
 301         my ($mb, $delta) = @_;
 302         return "0 B/s" if $mb <= 0;
 303         my $bw = int(($mb / $delta));
 304         return render_bytes($bw, 1) . "/s";
 305     };
 306
 307     my $target = 0;
 308     my $last_reused = 0;
 309     my $has_query_bitmap = $qemu_support && $qemu_support->{'query-bitmap-info'};
 310     my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
 311     if ($has_query_bitmap) {
 312         my $total = 0;
 313         my $bitmap_info = mon_cmd($vmid, 'query-pbs-bitmap-info');
 314         for my $info (sort { $a->{drive} cmp $b->{drive} } @$bitmap_info) {
 315             if (!$is_template) {
 316                 my $text = $bitmap_action_to_human->($self, $info);
 317                 my $drive = $info->{drive};
 318                 $drive =~ s/^drive-//; # for consistency
 319                 $self->loginfo("$drive: dirty-bitmap status: $text");
 320             }
 321             $target += $info->{dirty};
 322             $total += $info->{size};
 323             $last_reused += $info->{size} - $info->{dirty};
 324         }
 325         if ($target < $total) {
 326             my $total_h = render_bytes($total, 1);
 327             my $target_h = render_bytes($target, 1);
 328             $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total");
 329         }
 330     }
 331
 332     my $first_round = 1;
 333     my $last_finishing = 0;
 334     while(1) {
 335         my $status = mon_cmd($vmid, 'query-backup');
 336
 337         my $total = $status->{total} || 0;
 338         my $dirty = $status->{dirty};
 339         $target = (defined($dirty) && $dirty < $total) ? $dirty : $total if !$has_query_bitmap;
 340         $transferred = $status->{transferred} || 0;
 341         $reused = $status->{reused};
 342         my $percent = $target ? int(($transferred * 100)/$target) : 100;
 343         my $zero = $status->{'zero-bytes'} || 0;
 344
 345         die "got unexpected uuid\n" if !$status->{uuid} || ($status->{uuid} ne $job_uuid);
 346
 347         my $ctime = time();
 348         my $duration = $ctime - $starttime;
 349
 350         my $rbytes = $transferred - $last_transferred;
 351         my $wbytes;
 352         if ($reused) {
 353             # reused includes zero bytes for PBS
 354             $wbytes = $rbytes - ($reused - $last_reused);
 355         } else {
 356             $wbytes = $rbytes - ($zero - $last_zero);
 357         }
 358
 359         my $timediff = ($ctime - $last_time) || 1; # fixme
 360         my $mbps_read = $get_mbps->($rbytes, $timediff);
 361         my $mbps_write = $get_mbps->($wbytes, $timediff);
 362         my $target_h = render_bytes($target, 1);
 363         my $transferred_h = render_bytes($transferred, 1);
 364
 365         if (!$has_query_bitmap && $first_round && $target != $total) { # FIXME: remove with PVE 7.0
 366             my $total_h = render_bytes($total, 1);
 367             $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total");
 368         }
 369
 370         my $statusline = sprintf("%3d%% ($transferred_h of $target_h) in %s"
 371             .", read: $mbps_read, write: $mbps_write", $percent, render_duration($duration));
 372
 373         my $res = $status->{status} || 'unknown';
 374         if ($res ne 'active') {
 375             if ($last_percent < 100) {
 376                 $self->loginfo($statusline);
 377             }
 378             if ($res ne 'done') {
 379                 die (($status->{errmsg} || "unknown error") . "\n") if $res eq 'error';
 380                 die "got unexpected status '$res'\n";
 381             }
 382             $last_target = $target if $target;
 383             $last_total = $total if $total;
 384             $last_zero = $zero if $zero;
 385             $last_transferred = $transferred if $transferred;
 386             last;
 387         }
 388         if ($percent != $last_percent && ($timediff > 2)) {
 389             $self->loginfo($statusline);
 390             $last_percent = $percent;
 391             $last_target = $target if $target;
 392             $last_total = $total if $total;
 393             $last_zero = $zero if $zero;
 394             $last_transferred = $transferred if $transferred;
 395             $last_time = $ctime;
 396             $last_reused = $reused;
 397
 398             if (!$last_finishing && $status->{finishing}) {
 399                 $self->loginfo("Waiting for server to finish backup validation...");
 400             }
 401             $last_finishing = $status->{finishing};
 402         }
 403         sleep(1);
 404         $first_round = 0 if $first_round;
 405     }
 406
 407     my $duration = time() - $starttime;
 408
 409     if ($last_zero) {
 410         my $zero_per = $last_target ? int(($last_zero * 100)/$last_target) : 0;
 411         my $zero_h = render_bytes($last_zero);
 412         $self->loginfo("backup is sparse: $zero_h (${zero_per}%) total zero data");
 413     }
 414     if ($reused) {
 415         my $reused_h = render_bytes($reused);
 416         my $reuse_per = int($reused * 100 / $last_total);
 417         $self->loginfo("backup was done incrementally, reused $reused_h (${reuse_per}%)");
 418     }
 419     if ($transferred) {
 420         my $transferred_h = render_bytes($transferred);
 421         if ($duration) {
 422             my $mbps = $get_mbps->($transferred, $duration);
 423             $self->loginfo("transferred $transferred_h in $duration seconds ($mbps)");
 424         } else {
 425             $self->loginfo("transferred $transferred_h in <1 seconds");
 426         }
 427     }
 428
 429     return {
 430         total => $last_total,
 431         reused => $reused,
 432     };
 433 };
 434
 435 sub archive_pbs {
 436     my ($self, $task, $vmid) = @_;
 437
 438     my $conffile = "$task->{tmpdir}/qemu-server.conf";
 439     my $firewall = "$task->{tmpdir}/qemu-server.fw";
 440
 441     my $opts = $self->{vzdump}->{opts};
 442     my $scfg = $opts->{scfg};
 443
 444     my $starttime = time();
 445
 446     my $fingerprint = $scfg->{fingerprint};
 447     my $repo = PVE::PBSClient::get_repository($scfg);
 448     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $opts->{storage});
 449     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($scfg, $opts->{storage});
 450
 451     my $diskcount = scalar(@{$task->{disks}});
 452     # proxmox-backup-client can only handle raw files and block devs
 453     # only use it (directly) for disk-less VMs
 454     if (!$diskcount) {
 455         my @pathlist;
 456         $self->loginfo("backup contains no disks");
 457
 458         local $ENV{PBS_PASSWORD} = $password;
 459         local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
 460         my $cmd = [
 461             '/usr/bin/proxmox-backup-client',
 462             'backup',
 463             '--repository', $repo,
 464             '--backup-type', 'vm',
 465             '--backup-id', "$vmid",
 466             '--backup-time', $task->{backup_time},
 467         ];
 468
 469         push @$cmd, "qemu-server.conf:$conffile";
 470         push @$cmd, "fw.conf:$firewall" if -e $firewall;
 471
 472         $self->loginfo("starting template backup");
 473         $self->loginfo(join(' ', @$cmd));
 474
 475         $self->cmd($cmd);
 476
 477         return;
 478     }
 479
 480     # get list early so we die on unkown drive types before doing anything
 481     my $devlist = _get_task_devlist($task);
 482
 483     $self->enforce_vm_running_for_backup($vmid);
 484     $self->register_qmeventd_handle($vmid);
 485
 486     my $backup_job_uuid;
 487     eval {
 488         $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
 489             die "interrupted by signal\n";
 490         };
 491
 492         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
 493         if (!$qemu_support) {
 494             die "PBS backups are not supported by the running QEMU version. Please make "
 495               . "sure you've installed the latest version and the VM has been restarted.\n";
 496         }
 497
 498         my $fs_frozen = $self->qga_fs_freeze($task, $vmid);
 499
 500         my $params = {
 501             format => "pbs",
 502             'backup-file' => $repo,
 503             'backup-id' => "$vmid",
 504             'backup-time' => $task->{backup_time},
 505             password => $password,
 506             devlist => $devlist,
 507             'config-file' => $conffile,
 508         };
 509         $params->{speed} = $opts->{bwlimit}*1024 if $opts->{bwlimit};
 510         $params->{fingerprint} = $fingerprint if defined($fingerprint);
 511         $params->{'firewall-file'} = $firewall if -e $firewall;
 512         if (-e $keyfile) {
 513             $self->loginfo("enabling encryption");
 514             $params->{keyfile} = $keyfile;
 515             $params->{encrypt} = JSON::true;
 516         } else {
 517             $params->{encrypt} = JSON::false;
 518         }
 519
 520         my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
 521         $params->{'use-dirty-bitmap'} = JSON::true
 522             if $qemu_support->{'pbs-dirty-bitmap'} && !$is_template;
 523
 524         $params->{timeout} = 60; # give some time to connect to the backup server
 525
 526         my $res = eval { mon_cmd($vmid, "backup", %$params) };
 527         my $qmperr = $@;
 528         $backup_job_uuid = $res->{UUID} if $res;
 529
 530         if ($fs_frozen) {
 531             $self->qga_fs_thaw($vmid);
 532         }
 533
 534         die $qmperr if $qmperr;
 535         die "got no uuid for backup task\n" if !defined($backup_job_uuid);
 536
 537         $self->loginfo("started backup task '$backup_job_uuid'");
 538
 539         $self->resume_vm_after_job_start($task, $vmid);
 540
 541         my $stat = $query_backup_status_loop->($self, $vmid, $backup_job_uuid, $qemu_support);
 542         $task->{size} = $stat->{total};
 543     };
 544     my $err = $@;
 545     if ($err) {
 546         $self->logerr($err);
 547         $self->mon_backup_cancel($vmid);
 548     }
 549     $self->restore_vm_power_state($vmid);
 550
 551     die $err if $err;
 552 }
 553
 554 my $fork_compressor_pipe = sub {
 555     my ($self, $comp, $outfileno) = @_;
 556
 557     my @pipefd = POSIX::pipe();
 558     my $cpid = fork();
 559     die "unable to fork worker - $!" if !defined($cpid) || $cpid < 0;
 560     if ($cpid == 0) {
 561         eval {
 562             POSIX::close($pipefd[1]);
 563             # redirect STDIN
 564             my $fd = fileno(STDIN);
 565             close STDIN;
 566             POSIX::close(0) if $fd != 0;
 567             die "unable to redirect STDIN - $!"
 568                 if !open(STDIN, "<&", $pipefd[0]);
 569
 570             # redirect STDOUT
 571             $fd = fileno(STDOUT);
 572             close STDOUT;
 573             POSIX::close (1) if $fd != 1;
 574
 575             die "unable to redirect STDOUT - $!"
 576                 if !open(STDOUT, ">&", $outfileno);
 577
 578             exec($comp);
 579             die "fork compressor '$comp' failed\n";
 580         };
 581         if (my $err = $@) {
 582             $self->logerr($err);
 583             POSIX::_exit(1);
 584         }
 585         POSIX::_exit(0);
 586         kill(-9, $$);
 587     } else {
 588         POSIX::close($pipefd[0]);
 589         $outfileno = $pipefd[1];
 590     }
 591
 592     return ($cpid, $outfileno);
 593 };
 594
 595 sub archive_vma {
 596     my ($self, $task, $vmid, $filename, $comp) = @_;
 597
 598     my $conffile = "$task->{tmpdir}/qemu-server.conf";
 599     my $firewall = "$task->{tmpdir}/qemu-server.fw";
 600
 601     my $opts = $self->{vzdump}->{opts};
 602
 603     my $starttime = time();
 604
 605     my $speed = 0;
 606     if ($opts->{bwlimit}) {
 607         $speed = $opts->{bwlimit}*1024;
 608     }
 609
 610     my $diskcount = scalar(@{$task->{disks}});
 611     if (PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}) || !$diskcount) {
 612         my @pathlist;
 613         foreach my $di (@{$task->{disks}}) {
 614             if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 615                 push @pathlist, "$di->{qmdevice}=$di->{path}";
 616             } else {
 617                 die "implement me";
 618             }
 619         }
 620
 621         if (!$diskcount) {
 622             $self->loginfo("backup contains no disks");
 623         }
 624
 625         my $outcmd;
 626         if ($comp) {
 627             $outcmd = "exec:$comp";
 628         } else {
 629             $outcmd = "exec:cat";
 630         }
 631
 632         $outcmd .= " > $filename" if !$opts->{stdout};
 633
 634         my $cmd = ['/usr/bin/vma', 'create', '-v', '-c', $conffile];
 635         push @$cmd, '-c', $firewall if -e $firewall;
 636         push @$cmd, $outcmd, @pathlist;
 637
 638         $self->loginfo("starting template backup");
 639         $self->loginfo(join(' ', @$cmd));
 640
 641         if ($opts->{stdout}) {
 642             $self->cmd($cmd, output => ">&" . fileno($opts->{stdout}));
 643         } else {
 644             $self->cmd($cmd);
 645         }
 646
 647         return;
 648     }
 649
 650     my $devlist = _get_task_devlist($task);
 651
 652     $self->enforce_vm_running_for_backup($vmid);
 653     $self->register_qmeventd_handle($vmid);
 654
 655     my $cpid;
 656     my $backup_job_uuid;
 657
 658     eval {
 659         $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
 660             die "interrupted by signal\n";
 661         };
 662
 663         my $outfh;
 664         if ($opts->{stdout}) {
 665             $outfh = $opts->{stdout};
 666         } else {
 667             $outfh = IO::File->new($filename, "w") ||
 668                 die "unable to open file '$filename' - $!\n";
 669         }
 670         my $outfileno = fileno($outfh);
 671
 672         if ($comp) {
 673             ($cpid, $outfileno) = $fork_compressor_pipe->($self, $comp, $outfileno);
 674         }
 675
 676         my $qmpclient = PVE::QMPClient->new();
 677         my $backup_cb = sub {
 678             my ($vmid, $resp) = @_;
 679             $backup_job_uuid = $resp->{return}->{UUID};
 680         };
 681         my $add_fd_cb = sub {
 682             my ($vmid, $resp) = @_;
 683
 684             my $params = {
 685                 'backup-file' => "/dev/fdname/backup",
 686                 speed => $speed,
 687                 'config-file' => $conffile,
 688                 devlist => $devlist
 689             };
 690             $params->{'firewall-file'} = $firewall if -e $firewall;
 691
 692             $qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params);
 693         };
 694
 695         $qmpclient->queue_cmd($vmid, $add_fd_cb, 'getfd', fd => $outfileno, fdname => "backup");
 696
 697         my $fs_frozen = $self->qga_fs_freeze($task, $vmid);
 698
 699         eval { $qmpclient->queue_execute(30) };
 700         my $qmperr = $@;
 701
 702         if ($fs_frozen) {
 703             $self->qga_fs_thaw($vmid);
 704         }
 705
 706         die $qmperr if $qmperr;
 707         die $qmpclient->{errors}->{$vmid} if $qmpclient->{errors}->{$vmid};
 708
 709         if ($cpid) {
 710             POSIX::close($outfileno) == 0 ||
 711                 die "close output file handle failed\n";
 712         }
 713
 714         die "got no uuid for backup task\n" if !defined($backup_job_uuid);
 715
 716         $self->loginfo("started backup task '$backup_job_uuid'");
 717
 718         $self->resume_vm_after_job_start($task, $vmid);
 719
 720         $query_backup_status_loop->($self, $vmid, $backup_job_uuid);
 721     };
 722     my $err = $@;
 723     if ($err) {
 724         $self->logerr($err);
 725         $self->mon_backup_cancel($vmid);
 726     }
 727
 728     $self->restore_vm_power_state($vmid);
 729
 730     if ($err) {
 731         if ($cpid) {
 732             kill(9, $cpid);
 733             waitpid($cpid, 0);
 734         }
 735         die $err;
 736     }
 737
 738     if ($cpid && (waitpid($cpid, 0) > 0)) {
 739         my $stat = $?;
 740         my $ec = $stat >> 8;
 741         my $signal = $stat & 127;
 742         if ($ec || $signal) {
 743             die "$comp failed - wrong exit status $ec" .
 744                 ($signal ? " (signal $signal)\n" : "\n");
 745         }
 746     }
 747 }
 748
 749 sub _get_task_devlist {
 750     my ($task) = @_;
 751
 752     my $devlist = '';
 753     foreach my $di (@{$task->{disks}}) {
 754         if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 755             $devlist .= ',' if $devlist;
 756             $devlist .= $di->{qmdevice};
 757         } else {
 758             die "implement me (type '$di->{type}')";
 759         }
 760     }
 761     return $devlist;
 762 }
 763
 764 sub qga_fs_freeze {
 765     my ($self, $task, $vmid) = @_;
 766     return if !$self->{vmlist}->{$vmid}->{agent} || $task->{mode} eq 'stop' || !$self->{vm_was_running} || $self->{vm_was_paused};
 767
 768     if (!PVE::QemuServer::qga_check_running($vmid, 1)) {
 769         $self->loginfo("skipping guest-agent 'fs-freeze', agent configured but not running?");
 770         return;
 771     }
 772
 773     $self->loginfo("issuing guest-agent 'fs-freeze' command");
 774     eval { mon_cmd($vmid, "guest-fsfreeze-freeze") };
 775     $self->logerr($@) if $@;
 776
 777     return 1; # even on mon command error, ensure we always thaw again
 778 }
 779
 780 # only call if fs_freeze return 1
 781 sub qga_fs_thaw {
 782     my ($self, $vmid) = @_;
 783
 784     $self->loginfo("issuing guest-agent 'fs-thaw' command");
 785     eval { mon_cmd($vmid, "guest-fsfreeze-thaw") };
 786     $self->logerr($@) if $@;
 787 }
 788
 789 # we need a running QEMU/KVM process for backup, starts a paused (prelaunch)
 790 # one if VM isn't already running
 791 sub enforce_vm_running_for_backup {
 792     my ($self, $vmid) = @_;
 793
 794     if (PVE::QemuServer::check_running($vmid)) {
 795         $self->{vm_was_running} = 1;
 796         return;
 797     }
 798
 799     eval {
 800         $self->loginfo("starting kvm to execute backup task");
 801         # start with skiplock
 802         my $params = {
 803             skiplock => 1,
 804             skiptemplate => 1,
 805             paused => 1,
 806         };
 807         PVE::QemuServer::vm_start($self->{storecfg}, $vmid, $params);
 808     };
 809     die $@ if $@;
 810 }
 811
 812 sub register_qmeventd_handle {
 813     my ($self, $vmid) = @_;
 814
 815     my $fh;
 816     my $peer = "/var/run/qmeventd.sock";
 817     my $count = 0;
 818
 819     for (;;) {
 820         $count++;
 821         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
 822         last if $fh;
 823         if ($! != EINTR && $! != EAGAIN) {
 824             $self->log("warn", "unable to connect to qmeventd socket (vmid: $vmid) - $!\n");
 825             return;
 826         }
 827         if ($count > 4) {
 828             $self->log("warn", "unable to connect to qmeventd socket (vmid: $vmid)"
 829                              . " - timeout after $count retries\n");
 830             return;
 831         }
 832         usleep(25000);
 833     }
 834
 835     # send handshake to mark VM as backing up
 836     print $fh to_json({vzdump => {vmid => "$vmid"}});
 837     $self->{qmeventd_fh} = $fh;
 838 }
 839
 840 # resume VM againe once we got in a clear state (stop mode backup of running VM)
 841 sub resume_vm_after_job_start {
 842     my ($self, $task, $vmid) = @_;
 843
 844     return if !$self->{vm_was_running} || $self->{vm_was_paused};
 845
 846     if (my $stoptime = $task->{vmstoptime}) {
 847         my $delay = time() - $task->{vmstoptime};
 848         $task->{vmstoptime} = undef; # avoid printing 'online after ..' twice
 849         $self->loginfo("resuming VM again after $delay seconds");
 850     } else {
 851         $self->loginfo("resuming VM again");
 852     }
 853     mon_cmd($vmid, 'cont');
 854 }
 855
 856 # stop again if VM was not running before
 857 sub restore_vm_power_state {
 858     my ($self, $vmid) = @_;
 859
 860     # we always let VMs keep running
 861     return if $self->{vm_was_running};
 862
 863     eval {
 864         my $resp = mon_cmd($vmid, 'query-status');
 865         my $status = $resp && $resp->{status} ?  $resp->{status} : 'unknown';
 866         if ($status eq 'prelaunch') {
 867             $self->loginfo("stopping kvm after backup task");
 868             PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1);
 869         } else {
 870             $self->loginfo("kvm status changed after backup ('$status') - keep VM running");
 871         }
 872     };
 873     warn $@ if $@;
 874 }
 875
 876 sub mon_backup_cancel {
 877     my ($self, $vmid) = @_;
 878
 879     $self->loginfo("aborting backup job");
 880     eval { mon_cmd($vmid, 'backup-cancel') };
 881     $self->logerr($@) if $@;
 882 }
 883
 884 sub snapshot {
 885     my ($self, $task, $vmid) = @_;
 886
 887     # nothing to do
 888 }
 889
 890 sub cleanup {
 891     my ($self, $task, $vmid) = @_;
 892
 893     if ($self->{qmeventd_fh}) {
 894         close($self->{qmeventd_fh});
 895     }
 896 }
 897
 898 1;