PVE/VZDump/QemuServer.pm

   1 package PVE::VZDump::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Basename;
   7 use File::Path;
   8 use IO::File;
   9 use IPC::Open3;
  10 use JSON;
  11
  12 use PVE::Cluster qw(cfs_read_file);
  13 use PVE::INotify;
  14 use PVE::IPCC;
  15 use PVE::JSONSchema;
  16 use PVE::QMPClient;
  17 use PVE::Storage::Plugin;
  18 use PVE::Storage::PBSPlugin;
  19 use PVE::Storage;
  20 use PVE::Tools;
  21 use PVE::VZDump;
  22
  23 use PVE::QemuConfig;
  24 use PVE::QemuServer;
  25 use PVE::QemuServer::Machine;
  26 use PVE::QemuServer::Monitor qw(mon_cmd);
  27
  28 use base qw (PVE::VZDump::Plugin);
  29
  30 sub new {
  31     my ($class, $vzdump) = @_;
  32
  33     PVE::VZDump::check_bin('qm');
  34
  35     my $self = bless { vzdump => $vzdump }, $class;
  36
  37     $self->{vmlist} = PVE::QemuServer::vzlist();
  38     $self->{storecfg} = PVE::Storage::config();
  39
  40     return $self;
  41 };
  42
  43 sub type {
  44     return 'qemu';
  45 }
  46
  47 sub vmlist {
  48     my ($self) = @_;
  49     return [ keys %{$self->{vmlist}} ];
  50 }
  51
  52 sub prepare {
  53     my ($self, $task, $vmid, $mode) = @_;
  54
  55     $task->{disks} = [];
  56
  57     my $conf = $self->{vmlist}->{$vmid} = PVE::QemuConfig->load_config($vmid);
  58
  59     $self->loginfo("VM Name: $conf->{name}")
  60         if defined($conf->{name});
  61
  62     $self->{vm_was_running} = 1;
  63     if (!PVE::QemuServer::check_running($vmid)) {
  64         $self->{vm_was_running} = 0;
  65     }
  66
  67     $task->{hostname} = $conf->{name};
  68
  69     my $hostname = PVE::INotify::nodename();
  70
  71     my $vollist = [];
  72     my $drivehash = {};
  73     my $backup_volumes = PVE::QemuConfig->get_backup_volumes($conf);
  74
  75     foreach my $volume (@{$backup_volumes}) {
  76         my $name = $volume->{key};
  77         my $volume_config = $volume->{volume_config};
  78         my $volid = $volume_config->{file};
  79
  80         if (!$volume->{included}) {
  81             $self->loginfo("exclude disk '$name' '$volid' ($volume->{reason})");
  82             next;
  83         } elsif ($self->{vm_was_running} && $volume_config->{iothread}) {
  84             if (!PVE::QemuServer::Machine::runs_at_least_qemu_version($vmid, 4, 0, 1)) {
  85                 die "disk '$name' '$volid' (iothread=on) can't use backup feature with running QEMU " .
  86                     "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n";
  87             }
  88         } else {
  89             my $log = "include disk '$name' '$volid'";
  90             if (defined(my $size = $volume_config->{size})) {
  91                 my $readable_size = PVE::JSONSchema::format_size($size);
  92                 $log .= " $readable_size";
  93             }
  94             $self->loginfo($log);
  95         }
  96
  97         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
  98         push @$vollist, $volid if $storeid;
  99         $drivehash->{$name} = $volume->{volume_config};
 100     }
 101
 102     PVE::Storage::activate_volumes($self->{storecfg}, $vollist);
 103
 104     foreach my $ds (sort keys %$drivehash) {
 105         my $drive = $drivehash->{$ds};
 106
 107         my $volid = $drive->{file};
 108         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 109
 110         my $path = $volid;
 111         if ($storeid) {
 112             $path = PVE::Storage::path($self->{storecfg}, $volid);
 113         }
 114         next if !$path;
 115
 116         my ($size, $format) = eval { PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5) };
 117         die "no such volume '$volid'\n" if $@;
 118
 119         my $diskinfo = {
 120             path => $path,
 121             volid => $volid,
 122             storeid => $storeid,
 123             format => $format,
 124             virtdev => $ds,
 125             qmdevice => "drive-$ds",
 126         };
 127
 128         if (-b $path) {
 129             $diskinfo->{type} = 'block';
 130         } else {
 131             $diskinfo->{type} = 'file';
 132         }
 133
 134         push @{$task->{disks}}, $diskinfo;
 135     }
 136 }
 137
 138 sub vm_status {
 139     my ($self, $vmid) = @_;
 140
 141     my $running = PVE::QemuServer::check_running($vmid) ? 1 : 0;
 142
 143     return wantarray ? ($running, $running ? 'running' : 'stopped') : $running;
 144 }
 145
 146 sub lock_vm {
 147     my ($self, $vmid) = @_;
 148
 149     PVE::QemuConfig->set_lock($vmid, 'backup');
 150 }
 151
 152 sub unlock_vm {
 153     my ($self, $vmid) = @_;
 154
 155     PVE::QemuConfig->remove_lock($vmid, 'backup');
 156 }
 157
 158 sub stop_vm {
 159     my ($self, $task, $vmid) = @_;
 160
 161     my $opts = $self->{vzdump}->{opts};
 162
 163     my $wait = $opts->{stopwait} * 60;
 164     # send shutdown and wait
 165     $self->cmd ("qm shutdown $vmid --skiplock --keepActive --timeout $wait");
 166 }
 167
 168 sub start_vm {
 169     my ($self, $task, $vmid) = @_;
 170
 171     $self->cmd ("qm start $vmid --skiplock");
 172 }
 173
 174 sub suspend_vm {
 175     my ($self, $task, $vmid) = @_;
 176
 177     $self->cmd ("qm suspend $vmid --skiplock");
 178 }
 179
 180 sub resume_vm {
 181     my ($self, $task, $vmid) = @_;
 182
 183     $self->cmd ("qm resume $vmid --skiplock");
 184 }
 185
 186 sub assemble {
 187     my ($self, $task, $vmid) = @_;
 188
 189     my $conffile = PVE::QemuConfig->config_file($vmid);
 190
 191     my $outfile = "$task->{tmpdir}/qemu-server.conf";
 192     my $firewall_src = "/etc/pve/firewall/$vmid.fw";
 193     my $firewall_dest = "$task->{tmpdir}/qemu-server.fw";
 194
 195     my $outfd = IO::File->new (">$outfile") ||
 196         die "unable to open '$outfile'";
 197     my $conffd = IO::File->new ($conffile, 'r') ||
 198         die "unable open '$conffile'";
 199
 200     my $found_snapshot;
 201     my $found_pending;
 202     while (defined (my $line = <$conffd>)) {
 203         next if $line =~ m/^\#vzdump\#/; # just to be sure
 204         next if $line =~ m/^\#qmdump\#/; # just to be sure
 205         if ($line =~ m/^\[(.*)\]\s*$/) {
 206             if ($1 =~ m/PENDING/i) {
 207                 $found_pending = 1;
 208             } else {
 209                 $found_snapshot = 1;
 210             }
 211         }
 212         next if $found_snapshot || $found_pending; # skip all snapshots and pending changes config data
 213
 214         if ($line =~ m/^unused\d+:\s*(\S+)\s*/) {
 215             $self->loginfo("skip unused drive '$1' (not included into backup)");
 216             next;
 217         }
 218         next if $line =~ m/^lock:/ || $line =~ m/^parent:/;
 219
 220         print $outfd $line;
 221     }
 222
 223     foreach my $di (@{$task->{disks}}) {
 224         if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 225             my $storeid = $di->{storeid} || '';
 226             my $format = $di->{format} || '';
 227             print $outfd "#qmdump#map:$di->{virtdev}:$di->{qmdevice}:$storeid:$format:\n";
 228         } else {
 229             die "internal error";
 230         }
 231     }
 232
 233     if ($found_snapshot) {
 234         $self->loginfo("snapshots found (not included into backup)");
 235     }
 236     if ($found_pending) {
 237         $self->loginfo("pending configuration changes found (not included into backup)");
 238     }
 239
 240     PVE::Tools::file_copy($firewall_src, $firewall_dest) if -f $firewall_src;
 241 }
 242
 243 sub archive {
 244     my ($self, $task, $vmid, $filename, $comp) = @_;
 245
 246     my $opts = $self->{vzdump}->{opts};
 247     my $scfg = $opts->{scfg};
 248
 249     if ($self->{vzdump}->{opts}->{pbs}) {
 250         $self->archive_pbs($task, $vmid);
 251     } else {
 252         $self->archive_vma($task, $vmid, $filename, $comp);
 253     }
 254 }
 255
 256 # number, [precision=1]
 257 my $num2str = sub {
 258     return sprintf( "%." . ( $_[1] || 1 ) . "f", $_[0] );
 259 };
 260 my sub bytes_to_human {
 261     my ($bytes, $precission) = @_;
 262
 263     return $num2str->($bytes, $precission) . ' B' if $bytes < 1024;
 264     my $kb = $bytes/1024;
 265
 266     return $num2str->($kb, $precission) . " KiB" if $kb < 1024;
 267     my $mb = $kb/1024;
 268
 269     return $num2str->($mb, $precission) . " MiB" if $mb < 1024;
 270     my $gb = $mb/1024;
 271
 272     return $num2str->($gb, $precission) . " GiB" if $gb < 1024;
 273     my $tb = $gb/1024;
 274
 275     return $num2str->($tb, $precission) . " TiB";
 276 }
 277 my sub duration_to_human {
 278     my ($seconds) = @_;
 279
 280     return sprintf('%2ds', $seconds) if $seconds < 60;
 281     my $minutes = $seconds / 60;
 282     $seconds = $seconds % 60;
 283
 284     return sprintf('%2dm %2ds', $minutes, $seconds) if $minutes < 60;
 285     my $hours = $minutes / 60;
 286     $minutes = $minutes % 60;
 287
 288     return sprintf('%2dh %2dm %2ds', $hours, $minutes, $seconds) if $hours < 24;
 289     my $days = $hours / 24;
 290     $hours = $hours % 24;
 291
 292     return sprintf('%2dd %2dh %2dm', $days, $hours, $minutes);
 293 }
 294
 295 my $bitmap_action_to_human = sub {
 296     my ($self, $info) = @_;
 297
 298     my $action = $info->{action};
 299
 300     if ($action eq "not-used") {
 301         return "disabled (no support)" if $self->{vm_was_running};
 302         return "disabled (VM not running)";
 303     } elsif ($action eq "not-used-removed") {
 304         return "disabled (old bitmap cleared)";
 305     } elsif ($action eq "new") {
 306         return "created new";
 307     } elsif ($action eq "used") {
 308         if ($info->{dirty} == 0) {
 309             return "OK (drive clean)";
 310         } else {
 311             my $size = bytes_to_human($info->{size});
 312             my $dirty = bytes_to_human($info->{dirty});
 313             return "OK ($dirty of $size dirty)";
 314         }
 315     } elsif ($action eq "invalid") {
 316         return "existing bitmap was invalid and has been cleared";
 317     } else {
 318         return "unknown";
 319     }
 320 };
 321
 322 my $query_backup_status_loop = sub {
 323     my ($self, $vmid, $job_uuid, $qemu_support) = @_;
 324
 325     my $starttime = time ();
 326     my $last_time = $starttime;
 327     my ($last_percent, $last_total, $last_target, $last_zero, $last_transferred) = (-1, 0, 0, 0, 0);
 328     my ($transferred, $reused);
 329
 330     my $get_mbps = sub {
 331         my ($mb, $delta) = @_;
 332         return "0 B/s" if $mb <= 0;
 333         my $bw = int(($mb / $delta));
 334         return bytes_to_human($bw) . "/s";
 335     };
 336
 337     my $target = 0;
 338     my $last_reused = 0;
 339     my $has_query_bitmap = $qemu_support && $qemu_support->{'query-bitmap-info'};
 340     my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
 341     if ($has_query_bitmap) {
 342         my $total = 0;
 343         my $bitmap_info = mon_cmd($vmid, 'query-pbs-bitmap-info');
 344         for my $info (sort { $a->{drive} cmp $b->{drive} } @$bitmap_info) {
 345             if (!$is_template) {
 346                 my $text = $bitmap_action_to_human->($self, $info);
 347                 my $drive = $info->{drive};
 348                 $drive =~ s/^drive-//; # for consistency
 349                 $self->loginfo("$drive: dirty-bitmap status: $text");
 350             }
 351             $target += $info->{dirty};
 352             $total += $info->{size};
 353             $last_reused += $info->{size} - $info->{dirty};
 354         }
 355         if ($target < $total) {
 356             my $total_h = bytes_to_human($total);
 357             my $target_h = bytes_to_human($target);
 358             $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total");
 359         }
 360     }
 361
 362     my $first_round = 1;
 363     my $last_finishing = 0;
 364     while(1) {
 365         my $status = mon_cmd($vmid, 'query-backup');
 366
 367         my $total = $status->{total} || 0;
 368         my $dirty = $status->{dirty};
 369         $target = (defined($dirty) && $dirty < $total) ? $dirty : $total if !$has_query_bitmap;
 370         $transferred = $status->{transferred} || 0;
 371         $reused = $status->{reused};
 372         my $percent = $target ? int(($transferred * 100)/$target) : 100;
 373         my $zero = $status->{'zero-bytes'} || 0;
 374
 375         die "got unexpected uuid\n" if !$status->{uuid} || ($status->{uuid} ne $job_uuid);
 376
 377         my $ctime = time();
 378         my $duration = $ctime - $starttime;
 379
 380         my $rbytes = $transferred - $last_transferred;
 381         my $wbytes;
 382         if ($reused) {
 383             # reused includes zero bytes for PBS
 384             $wbytes = $rbytes - ($reused - $last_reused);
 385         } else {
 386             $wbytes = $rbytes - ($zero - $last_zero);
 387         }
 388
 389         my $timediff = ($ctime - $last_time) || 1; # fixme
 390         my $mbps_read = $get_mbps->($rbytes, $timediff);
 391         my $mbps_write = $get_mbps->($wbytes, $timediff);
 392         my $target_h = bytes_to_human($target);
 393         my $transferred_h = bytes_to_human($transferred);
 394
 395         if (!$has_query_bitmap && $first_round && $target != $total) { # FIXME: remove with PVE 7.0
 396             my $total_h = bytes_to_human($total);
 397             $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total");
 398         }
 399
 400         my $statusline = sprintf("%3d%% ($transferred_h of $target_h) in %s"
 401             .", read: $mbps_read, write: $mbps_write", $percent, duration_to_human($duration));
 402
 403         my $res = $status->{status} || 'unknown';
 404         if ($res ne 'active') {
 405             if ($last_percent < 100) {
 406                 $self->loginfo($statusline);
 407             }
 408             if ($res ne 'done') {
 409                 die (($status->{errmsg} || "unknown error") . "\n") if $res eq 'error';
 410                 die "got unexpected status '$res'\n";
 411             }
 412             $last_target = $target if $target;
 413             $last_total = $total if $total;
 414             $last_zero = $zero if $zero;
 415             $last_transferred = $transferred if $transferred;
 416             last;
 417         }
 418         if ($percent != $last_percent && ($timediff > 2)) {
 419             $self->loginfo($statusline);
 420             $last_percent = $percent;
 421             $last_target = $target if $target;
 422             $last_total = $total if $total;
 423             $last_zero = $zero if $zero;
 424             $last_transferred = $transferred if $transferred;
 425             $last_time = $ctime;
 426             $last_reused = $reused;
 427
 428             if (!$last_finishing && $status->{finishing}) {
 429                 $self->loginfo("Waiting for server to finish backup validation...");
 430             }
 431             $last_finishing = $status->{finishing};
 432         }
 433         sleep(1);
 434         $first_round = 0 if $first_round;
 435     }
 436
 437     my $duration = time() - $starttime;
 438
 439     if ($last_zero) {
 440         my $zero_per = $last_target ? int(($last_zero * 100)/$last_target) : 0;
 441         my $zero_h = bytes_to_human($last_zero, 2);
 442         $self->loginfo("backup is sparse: $zero_h (${zero_per}%) total zero data");
 443     }
 444     if ($reused) {
 445         my $reused_h = bytes_to_human($reused, 2);
 446         my $reuse_per = int($reused * 100 / $last_total);
 447         $self->loginfo("backup was done incrementally, reused $reused_h (${reuse_per}%)");
 448     }
 449     if ($transferred) {
 450         my $transferred_h = bytes_to_human($transferred, 2);
 451         if ($duration) {
 452             my $mbps = $get_mbps->($transferred, $duration);
 453             $self->loginfo("transferred $transferred_h in $duration seconds ($mbps)");
 454         } else {
 455             $self->loginfo("transferred $transferred_h in <1 seconds");
 456         }
 457     }
 458
 459     return {
 460         total => $last_total,
 461         reused => $reused,
 462     };
 463 };
 464
 465 sub archive_pbs {
 466     my ($self, $task, $vmid) = @_;
 467
 468     my $conffile = "$task->{tmpdir}/qemu-server.conf";
 469     my $firewall = "$task->{tmpdir}/qemu-server.fw";
 470
 471     my $opts = $self->{vzdump}->{opts};
 472     my $scfg = $opts->{scfg};
 473
 474     my $starttime = time();
 475
 476     my $server = $scfg->{server};
 477     my $datastore = $scfg->{datastore};
 478     my $username = $scfg->{username} // 'root@pam';
 479     my $fingerprint = $scfg->{fingerprint};
 480
 481     my $repo = "$username\@$server:$datastore";
 482     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $opts->{storage});
 483     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($scfg, $opts->{storage});
 484
 485     my $diskcount = scalar(@{$task->{disks}});
 486     # proxmox-backup-client can only handle raw files and block devs
 487     # only use it (directly) for disk-less VMs
 488     if (!$diskcount) {
 489         my @pathlist;
 490         $self->loginfo("backup contains no disks");
 491
 492         local $ENV{PBS_PASSWORD} = $password;
 493         local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
 494         my $cmd = [
 495             '/usr/bin/proxmox-backup-client',
 496             'backup',
 497             '--repository', $repo,
 498             '--backup-type', 'vm',
 499             '--backup-id', "$vmid",
 500             '--backup-time', $task->{backup_time},
 501         ];
 502
 503         push @$cmd, "qemu-server.conf:$conffile";
 504         push @$cmd, "fw.conf:$firewall" if -e $firewall;
 505
 506         $self->loginfo("starting template backup");
 507         $self->loginfo(join(' ', @$cmd));
 508
 509         $self->cmd($cmd);
 510
 511         return;
 512     }
 513
 514     # get list early so we die on unkown drive types before doing anything
 515     my $devlist = _get_task_devlist($task);
 516
 517     $self->enforce_vm_running_for_backup($vmid);
 518
 519     my $backup_job_uuid;
 520     eval {
 521         $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
 522             die "interrupted by signal\n";
 523         };
 524
 525         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
 526         if (!$qemu_support) {
 527             die "PBS backups are not supported by the running QEMU version. Please make "
 528               . "sure you've installed the latest version and the VM has been restarted.\n";
 529         }
 530
 531         my $fs_frozen = $self->qga_fs_freeze($task, $vmid);
 532
 533         my $params = {
 534             format => "pbs",
 535             'backup-file' => $repo,
 536             'backup-id' => "$vmid",
 537             'backup-time' => $task->{backup_time},
 538             password => $password,
 539             devlist => $devlist,
 540             'config-file' => $conffile,
 541         };
 542         $params->{speed} = $opts->{bwlimit}*1024 if $opts->{bwlimit};
 543         $params->{fingerprint} = $fingerprint if defined($fingerprint);
 544         $params->{'firewall-file'} = $firewall if -e $firewall;
 545         if (-e $keyfile) {
 546             $self->loginfo("enabling encryption");
 547             $params->{keyfile} = $keyfile;
 548             $params->{encrypt} = JSON::true;
 549         } else {
 550             $params->{encrypt} = JSON::false;
 551         }
 552
 553         my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
 554         $params->{'use-dirty-bitmap'} = JSON::true
 555             if $qemu_support->{'pbs-dirty-bitmap'} && $self->{vm_was_running} && !$is_template;
 556
 557         $params->{timeout} = 60; # give some time to connect to the backup server
 558
 559         my $res = eval { mon_cmd($vmid, "backup", %$params) };
 560         my $qmperr = $@;
 561         $backup_job_uuid = $res->{UUID} if $res;
 562
 563         if ($fs_frozen) {
 564             $self->qga_fs_thaw($vmid);
 565         }
 566
 567         die $qmperr if $qmperr;
 568         die "got no uuid for backup task\n" if !defined($backup_job_uuid);
 569
 570         $self->loginfo("started backup task '$backup_job_uuid'");
 571
 572         $self->resume_vm_after_job_start($task, $vmid);
 573
 574         my $stat = $query_backup_status_loop->($self, $vmid, $backup_job_uuid, $qemu_support);
 575         $task->{size} = $stat->{total};
 576     };
 577     my $err = $@;
 578     if ($err) {
 579         $self->logerr($err);
 580         $self->mon_backup_cancel($vmid) if defined($backup_job_uuid);
 581     }
 582     $self->restore_vm_power_state($vmid);
 583
 584     die $err if $err;
 585 }
 586
 587 my $fork_compressor_pipe = sub {
 588     my ($self, $comp, $outfileno) = @_;
 589
 590     my @pipefd = POSIX::pipe();
 591     my $cpid = fork();
 592     die "unable to fork worker - $!" if !defined($cpid) || $cpid < 0;
 593     if ($cpid == 0) {
 594         eval {
 595             POSIX::close($pipefd[1]);
 596             # redirect STDIN
 597             my $fd = fileno(STDIN);
 598             close STDIN;
 599             POSIX::close(0) if $fd != 0;
 600             die "unable to redirect STDIN - $!"
 601                 if !open(STDIN, "<&", $pipefd[0]);
 602
 603             # redirect STDOUT
 604             $fd = fileno(STDOUT);
 605             close STDOUT;
 606             POSIX::close (1) if $fd != 1;
 607
 608             die "unable to redirect STDOUT - $!"
 609                 if !open(STDOUT, ">&", $outfileno);
 610
 611             exec($comp);
 612             die "fork compressor '$comp' failed\n";
 613         };
 614         if (my $err = $@) {
 615             $self->logerr($err);
 616             POSIX::_exit(1);
 617         }
 618         POSIX::_exit(0);
 619         kill(-9, $$);
 620     } else {
 621         POSIX::close($pipefd[0]);
 622         $outfileno = $pipefd[1];
 623     }
 624
 625     return ($cpid, $outfileno);
 626 };
 627
 628 sub archive_vma {
 629     my ($self, $task, $vmid, $filename, $comp) = @_;
 630
 631     my $conffile = "$task->{tmpdir}/qemu-server.conf";
 632     my $firewall = "$task->{tmpdir}/qemu-server.fw";
 633
 634     my $opts = $self->{vzdump}->{opts};
 635
 636     my $starttime = time();
 637
 638     my $speed = 0;
 639     if ($opts->{bwlimit}) {
 640         $speed = $opts->{bwlimit}*1024;
 641     }
 642
 643     my $diskcount = scalar(@{$task->{disks}});
 644     if (PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}) || !$diskcount) {
 645         my @pathlist;
 646         foreach my $di (@{$task->{disks}}) {
 647             if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 648                 push @pathlist, "$di->{qmdevice}=$di->{path}";
 649             } else {
 650                 die "implement me";
 651             }
 652         }
 653
 654         if (!$diskcount) {
 655             $self->loginfo("backup contains no disks");
 656         }
 657
 658         my $outcmd;
 659         if ($comp) {
 660             $outcmd = "exec:$comp";
 661         } else {
 662             $outcmd = "exec:cat";
 663         }
 664
 665         $outcmd .= " > $filename" if !$opts->{stdout};
 666
 667         my $cmd = ['/usr/bin/vma', 'create', '-v', '-c', $conffile];
 668         push @$cmd, '-c', $firewall if -e $firewall;
 669         push @$cmd, $outcmd, @pathlist;
 670
 671         $self->loginfo("starting template backup");
 672         $self->loginfo(join(' ', @$cmd));
 673
 674         if ($opts->{stdout}) {
 675             $self->cmd($cmd, output => ">&" . fileno($opts->{stdout}));
 676         } else {
 677             $self->cmd($cmd);
 678         }
 679
 680         return;
 681     }
 682
 683     my $devlist = _get_task_devlist($task);
 684
 685     $self->enforce_vm_running_for_backup($vmid);
 686
 687     my $cpid;
 688     my $backup_job_uuid;
 689
 690     eval {
 691         $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
 692             die "interrupted by signal\n";
 693         };
 694
 695         my $outfh;
 696         if ($opts->{stdout}) {
 697             $outfh = $opts->{stdout};
 698         } else {
 699             $outfh = IO::File->new($filename, "w") ||
 700                 die "unable to open file '$filename' - $!\n";
 701         }
 702         my $outfileno = fileno($outfh);
 703
 704         if ($comp) {
 705             ($cpid, $outfileno) = $fork_compressor_pipe->($self, $comp, $outfileno);
 706         }
 707
 708         my $qmpclient = PVE::QMPClient->new();
 709         my $backup_cb = sub {
 710             my ($vmid, $resp) = @_;
 711             $backup_job_uuid = $resp->{return}->{UUID};
 712         };
 713         my $add_fd_cb = sub {
 714             my ($vmid, $resp) = @_;
 715
 716             my $params = {
 717                 'backup-file' => "/dev/fdname/backup",
 718                 speed => $speed,
 719                 'config-file' => $conffile,
 720                 devlist => $devlist
 721             };
 722             $params->{'firewall-file'} = $firewall if -e $firewall;
 723
 724             $qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params);
 725         };
 726
 727         $qmpclient->queue_cmd($vmid, $add_fd_cb, 'getfd', fd => $outfileno, fdname => "backup");
 728
 729         my $fs_frozen = $self->qga_fs_freeze($task, $vmid);
 730
 731         eval { $qmpclient->queue_execute(30) };
 732         my $qmperr = $@;
 733
 734         if ($fs_frozen) {
 735             $self->qga_fs_thaw($vmid);
 736         }
 737
 738         die $qmperr if $qmperr;
 739         die $qmpclient->{errors}->{$vmid} if $qmpclient->{errors}->{$vmid};
 740
 741         if ($cpid) {
 742             POSIX::close($outfileno) == 0 ||
 743                 die "close output file handle failed\n";
 744         }
 745
 746         die "got no uuid for backup task\n" if !defined($backup_job_uuid);
 747
 748         $self->loginfo("started backup task '$backup_job_uuid'");
 749
 750         $self->resume_vm_after_job_start($task, $vmid);
 751
 752         $query_backup_status_loop->($self, $vmid, $backup_job_uuid);
 753     };
 754     my $err = $@;
 755     if ($err) {
 756         $self->logerr($err);
 757         $self->mon_backup_cancel($vmid) if defined($backup_job_uuid);
 758     }
 759
 760     $self->restore_vm_power_state($vmid);
 761
 762     if ($err) {
 763         if ($cpid) {
 764             kill(9, $cpid);
 765             waitpid($cpid, 0);
 766         }
 767         die $err;
 768     }
 769
 770     if ($cpid && (waitpid($cpid, 0) > 0)) {
 771         my $stat = $?;
 772         my $ec = $stat >> 8;
 773         my $signal = $stat & 127;
 774         if ($ec || $signal) {
 775             die "$comp failed - wrong exit status $ec" .
 776                 ($signal ? " (signal $signal)\n" : "\n");
 777         }
 778     }
 779 }
 780
 781 sub _get_task_devlist {
 782     my ($task) = @_;
 783
 784     my $devlist = '';
 785     foreach my $di (@{$task->{disks}}) {
 786         if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 787             $devlist .= ',' if $devlist;
 788             $devlist .= $di->{qmdevice};
 789         } else {
 790             die "implement me (type '$di->{type}')";
 791         }
 792     }
 793     return $devlist;
 794 }
 795
 796 sub qga_fs_freeze {
 797     my ($self, $task, $vmid) = @_;
 798     return if !$self->{vmlist}->{$vmid}->{agent} || $task->{mode} eq 'stop' || !$self->{vm_was_running};
 799
 800     if (!PVE::QemuServer::qga_check_running($vmid, 1)) {
 801         $self->loginfo("skipping guest-agent 'fs-freeze', agent configured but not running?");
 802         return;
 803     }
 804
 805     $self->loginfo("issuing guest-agent 'fs-freeze' command");
 806     eval { mon_cmd($vmid, "guest-fsfreeze-freeze") };
 807     $self->logerr($@) if $@;
 808
 809     return 1; # even on mon command error, ensure we always thaw again
 810 }
 811
 812 # only call if fs_freeze return 1
 813 sub qga_fs_thaw {
 814     my ($self, $vmid) = @_;
 815
 816     $self->loginfo("issuing guest-agent 'fs-thaw' command");
 817     eval { mon_cmd($vmid, "guest-fsfreeze-thaw") };
 818     $self->logerr($@) if $@;
 819 }
 820
 821 # we need a running QEMU/KVM process for backup, starts a paused (prelaunch)
 822 # one if VM isn't already running
 823 sub enforce_vm_running_for_backup {
 824     my ($self, $vmid) = @_;
 825
 826     if (PVE::QemuServer::check_running($vmid)) {
 827         $self->{vm_was_running} = 1;
 828         return;
 829     }
 830
 831     eval {
 832         $self->loginfo("starting kvm to execute backup task");
 833         # start with skiplock
 834         my $params = {
 835             skiplock => 1,
 836             skiptemplate => 1,
 837             paused => 1,
 838         };
 839         PVE::QemuServer::vm_start($self->{storecfg}, $vmid, $params);
 840     };
 841     die $@ if $@;
 842 }
 843
 844 # resume VM againe once we got in a clear state (stop mode backup of running VM)
 845 sub resume_vm_after_job_start {
 846     my ($self, $task, $vmid) = @_;
 847
 848     return if !$self->{vm_was_running};
 849
 850     if (my $stoptime = $task->{vmstoptime}) {
 851         my $delay = time() - $task->{vmstoptime};
 852         $task->{vmstoptime} = undef; # avoid printing 'online after ..' twice
 853         $self->loginfo("resuming VM again after $delay seconds");
 854     } else {
 855         $self->loginfo("resuming VM again");
 856     }
 857     mon_cmd($vmid, 'cont');
 858 }
 859
 860 # stop again if VM was not running before
 861 sub restore_vm_power_state {
 862     my ($self, $vmid) = @_;
 863
 864     # we always let VMs keep running
 865     return if $self->{vm_was_running};
 866
 867     eval {
 868         my $resp = mon_cmd($vmid, 'query-status');
 869         my $status = $resp && $resp->{status} ?  $resp->{status} : 'unknown';
 870         if ($status eq 'prelaunch') {
 871             $self->loginfo("stopping kvm after backup task");
 872             PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1);
 873         } else {
 874             $self->loginfo("kvm status changed after backup ('$status') - keep VM running");
 875         }
 876     };
 877     warn $@ if $@;
 878 }
 879
 880 sub mon_backup_cancel {
 881     my ($self, $vmid) = @_;
 882
 883     $self->loginfo("aborting backup job");
 884     eval { mon_cmd($vmid, 'backup-cancel') };
 885     $self->logerr($@) if $@;
 886 }
 887
 888 sub snapshot {
 889     my ($self, $task, $vmid) = @_;
 890
 891     # nothing to do
 892 }
 893
 894 sub cleanup {
 895     my ($self, $task, $vmid) = @_;
 896
 897     # nothing to do ?
 898 }
 899
 900 1;