PVE/VZDump/QemuServer.pm

   1 package PVE::VZDump::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Basename;
   7 use File::Path;
   8 use IO::File;
   9 use IPC::Open3;
  10
  11 use PVE::Cluster qw(cfs_read_file);
  12 use PVE::INotify;
  13 use PVE::IPCC;
  14 use PVE::JSONSchema;
  15 use PVE::QMPClient;
  16 use PVE::Storage::Plugin;
  17 use PVE::Storage;
  18 use PVE::Tools;
  19 use PVE::VZDump;
  20
  21 use PVE::QemuServer;
  22 use PVE::QemuServer::Machine;
  23 use PVE::QemuServer::Monitor qw(mon_cmd);
  24
  25 use base qw (PVE::VZDump::Plugin);
  26
  27 sub new {
  28     my ($class, $vzdump) = @_;
  29
  30     PVE::VZDump::check_bin('qm');
  31
  32     my $self = bless { vzdump => $vzdump }, $class;
  33
  34     $self->{vmlist} = PVE::QemuServer::vzlist();
  35     $self->{storecfg} = PVE::Storage::config();
  36
  37     return $self;
  38 };
  39
  40
  41 sub type {
  42     return 'qemu';
  43 }
  44
  45 sub vmlist {
  46     my ($self) = @_;
  47
  48     return [ keys %{$self->{vmlist}} ];
  49 }
  50
  51 sub prepare {
  52     my ($self, $task, $vmid, $mode) = @_;
  53
  54     $task->{disks} = [];
  55
  56     my $conf = $self->{vmlist}->{$vmid} = PVE::QemuConfig->load_config($vmid);
  57
  58     $self->loginfo("VM Name: $conf->{name}")
  59         if defined($conf->{name});
  60
  61     $self->{vm_was_running} = 1;
  62     if (!PVE::QemuServer::check_running($vmid)) {
  63         $self->{vm_was_running} = 0;
  64     }
  65
  66     $task->{hostname} = $conf->{name};
  67
  68     my $hostname = PVE::INotify::nodename();
  69
  70     my $vollist = [];
  71     my $drivehash = {};
  72     PVE::QemuServer::foreach_drive($conf, sub {
  73         my ($ds, $drive) = @_;
  74
  75         return if PVE::QemuServer::drive_is_cdrom($drive);
  76
  77         my $volid = $drive->{file};
  78
  79         if (defined($drive->{backup}) && !$drive->{backup}) {
  80             $self->loginfo("exclude disk '$ds' '$volid' (backup=no)");
  81             return;
  82         } elsif ($self->{vm_was_running} && $drive->{iothread}) {
  83             if (!PVE::QemuServer::Machine::runs_at_least_qemu_version($vmid, 4, 0, 1)) {
  84                 die "disk '$ds' '$volid' (iothread=on) can't use backup feature with running QEMU " .
  85                     "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n";
  86             }
  87         } elsif ($ds =~ m/^efidisk/ && (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf')) {
  88             $self->loginfo("excluding '$ds' (efidisks can only be backed up when BIOS is set to 'ovmf')");
  89             return;
  90         } else {
  91             my $log = "include disk '$ds' '$volid'";
  92            if (defined $drive->{size}) {
  93                 my $readable_size = PVE::JSONSchema::format_size($drive->{size});
  94                 $log .= " $readable_size";
  95            }
  96             $self->loginfo($log);
  97         }
  98
  99         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 100         push @$vollist, $volid if $storeid;
 101         $drivehash->{$ds} = $drive;
 102     });
 103
 104     PVE::Storage::activate_volumes($self->{storecfg}, $vollist);
 105
 106     foreach my $ds (sort keys %$drivehash) {
 107         my $drive = $drivehash->{$ds};
 108
 109         my $volid = $drive->{file};
 110
 111         my $path;
 112
 113         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 114         if ($storeid) {
 115             $path = PVE::Storage::path($self->{storecfg}, $volid);
 116         } else {
 117             $path = $volid;
 118         }
 119
 120         next if !$path;
 121
 122         my $format = undef;
 123         my $size = undef;
 124
 125         eval{
 126             ($size, $format) = PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5);
 127         };
 128         die "no such volume '$volid'\n" if $@;
 129
 130         my $diskinfo = { path => $path , volid => $volid, storeid => $storeid,
 131                          format => $format, virtdev => $ds, qmdevice => "drive-$ds" };
 132
 133         if (-b $path) {
 134             $diskinfo->{type} = 'block';
 135         } else {
 136             $diskinfo->{type} = 'file';
 137         }
 138
 139         push @{$task->{disks}}, $diskinfo;
 140     }
 141 }
 142
 143 sub vm_status {
 144     my ($self, $vmid) = @_;
 145
 146     my $running = PVE::QemuServer::check_running($vmid) ? 1 : 0;
 147
 148     return wantarray ? ($running, $running ? 'running' : 'stopped') : $running;
 149 }
 150
 151 sub lock_vm {
 152     my ($self, $vmid) = @_;
 153
 154     $self->cmd ("qm set $vmid --lock backup");
 155 }
 156
 157 sub unlock_vm {
 158     my ($self, $vmid) = @_;
 159
 160     $self->cmd ("qm unlock $vmid");
 161 }
 162
 163 sub stop_vm {
 164     my ($self, $task, $vmid) = @_;
 165
 166     my $opts = $self->{vzdump}->{opts};
 167
 168     my $wait = $opts->{stopwait} * 60;
 169     # send shutdown and wait
 170     $self->cmd ("qm shutdown $vmid --skiplock --keepActive --timeout $wait");
 171 }
 172
 173 sub start_vm {
 174     my ($self, $task, $vmid) = @_;
 175
 176     $self->cmd ("qm start $vmid --skiplock");
 177 }
 178
 179 sub suspend_vm {
 180     my ($self, $task, $vmid) = @_;
 181
 182     $self->cmd ("qm suspend $vmid --skiplock");
 183 }
 184
 185 sub resume_vm {
 186     my ($self, $task, $vmid) = @_;
 187
 188     $self->cmd ("qm resume $vmid --skiplock");
 189 }
 190
 191 sub assemble {
 192     my ($self, $task, $vmid) = @_;
 193
 194     my $conffile = PVE::QemuConfig->config_file($vmid);
 195
 196     my $outfile = "$task->{tmpdir}/qemu-server.conf";
 197     my $firewall_src = "/etc/pve/firewall/$vmid.fw";
 198     my $firewall_dest = "$task->{tmpdir}/qemu-server.fw";
 199
 200     my $outfd = IO::File->new (">$outfile") ||
 201         die "unable to open '$outfile'";
 202     my $conffd = IO::File->new ($conffile, 'r') ||
 203         die "unable open '$conffile'";
 204
 205     my $found_snapshot;
 206     my $found_pending;
 207     while (defined (my $line = <$conffd>)) {
 208         next if $line =~ m/^\#vzdump\#/; # just to be sure
 209         next if $line =~ m/^\#qmdump\#/; # just to be sure
 210         if ($line =~ m/^\[(.*)\]\s*$/) {
 211             if ($1 =~ m/PENDING/i) {
 212                 $found_pending = 1;
 213             } else {
 214                 $found_snapshot = 1;
 215             }
 216         }
 217
 218         next if $found_snapshot; # skip all snapshots data
 219         next if $found_pending; # skip all pending changes
 220
 221         if ($line =~ m/^unused\d+:\s*(\S+)\s*/) {
 222             $self->loginfo("skip unused drive '$1' (not included into backup)");
 223             next;
 224         }
 225         next if $line =~ m/^lock:/ || $line =~ m/^parent:/;
 226
 227         print $outfd $line;
 228     }
 229
 230     foreach my $di (@{$task->{disks}}) {
 231         if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 232             my $storeid = $di->{storeid} || '';
 233             my $format = $di->{format} || '';
 234             print $outfd "#qmdump#map:$di->{virtdev}:$di->{qmdevice}:$storeid:$format:\n";
 235         } else {
 236             die "internal error";
 237         }
 238     }
 239
 240     if ($found_snapshot) {
 241         $self->loginfo("snapshots found (not included into backup)");
 242     }
 243
 244     if ($found_pending) {
 245         $self->loginfo("pending configuration changes found (not included into backup)");
 246     }
 247
 248     PVE::Tools::file_copy($firewall_src, $firewall_dest) if -f $firewall_src;
 249 }
 250
 251 sub archive {
 252     my ($self, $task, $vmid, $filename, $comp) = @_;
 253
 254     my $conffile = "$task->{tmpdir}/qemu-server.conf";
 255     my $firewall = "$task->{tmpdir}/qemu-server.fw";
 256
 257     my $opts = $self->{vzdump}->{opts};
 258
 259     my $starttime = time ();
 260
 261     my $speed = 0;
 262     if ($opts->{bwlimit}) {
 263         $speed = $opts->{bwlimit}*1024;
 264     }
 265
 266     my $diskcount = scalar(@{$task->{disks}});
 267
 268     if (PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}) || !$diskcount) {
 269         my @pathlist;
 270         foreach my $di (@{$task->{disks}}) {
 271             if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 272                 push @pathlist, "$di->{qmdevice}=$di->{path}";
 273             } else {
 274                 die "implement me";
 275             }
 276         }
 277
 278         if (!$diskcount) {
 279             $self->loginfo("backup contains no disks");
 280         }
 281
 282         my $outcmd;
 283         if ($comp) {
 284             $outcmd = "exec:$comp";
 285         } else {
 286             $outcmd = "exec:cat";
 287         }
 288
 289         $outcmd .= " > $filename" if !$opts->{stdout};
 290
 291         my $cmd = ['/usr/bin/vma', 'create', '-v', '-c', $conffile];
 292         push @$cmd, '-c', $firewall if -e $firewall;
 293         push @$cmd, $outcmd, @pathlist;
 294
 295         $self->loginfo("starting template backup");
 296         $self->loginfo(join(' ', @$cmd));
 297
 298         if ($opts->{stdout}) {
 299             $self->cmd($cmd, output => ">&=" . fileno($opts->{stdout}));
 300         } else {
 301             $self->cmd($cmd);
 302         }
 303
 304         return;
 305     }
 306
 307
 308     my $devlist = '';
 309     foreach my $di (@{$task->{disks}}) {
 310         if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 311             $devlist .= $devlist ? ",$di->{qmdevice}" : $di->{qmdevice};
 312         } else {
 313             die "implement me";
 314         }
 315     }
 316
 317     my $stop_after_backup;
 318     my $resume_on_backup;
 319
 320     my $skiplock = 1;
 321     my $vm_is_running = PVE::QemuServer::check_running($vmid);
 322     if (!$vm_is_running) {
 323         eval {
 324             $self->loginfo("starting kvm to execute backup task");
 325             PVE::QemuServer::vm_start($self->{storecfg}, $vmid, undef,
 326                                       $skiplock, undef, 1);
 327             if ($self->{vm_was_running}) {
 328                 $resume_on_backup = 1;
 329             } else {
 330                 $stop_after_backup = 1;
 331             }
 332         };
 333         if (my $err = $@) {
 334             die $err;
 335         }
 336     }
 337
 338     my $cpid;
 339     my $interrupt_msg = "interrupted by signal\n";
 340     eval {
 341         $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
 342             die $interrupt_msg;
 343         };
 344
 345         my $qmpclient = PVE::QMPClient->new();
 346
 347         my $uuid;
 348
 349         my $backup_cb = sub {
 350             my ($vmid, $resp) = @_;
 351             $uuid = $resp->{return}->{UUID};
 352         };
 353
 354         my $outfh;
 355         if ($opts->{stdout}) {
 356             $outfh = $opts->{stdout};
 357         } else {
 358             $outfh = IO::File->new($filename, "w") ||
 359                 die "unable to open file '$filename' - $!\n";
 360         }
 361
 362         my $outfileno;
 363         if ($comp) {
 364             my @pipefd = POSIX::pipe();
 365             $cpid = fork();
 366             die "unable to fork worker - $!" if !defined($cpid);
 367             if ($cpid == 0) {
 368                 eval {
 369                     POSIX::close($pipefd[1]);
 370                     # redirect STDIN
 371                     my $fd = fileno(STDIN);
 372                     close STDIN;
 373                     POSIX::close(0) if $fd != 0;
 374                     die "unable to redirect STDIN - $!"
 375                         if !open(STDIN, "<&", $pipefd[0]);
 376
 377                     # redirect STDOUT
 378                     $fd = fileno(STDOUT);
 379                     close STDOUT;
 380                     POSIX::close (1) if $fd != 1;
 381
 382                     die "unable to redirect STDOUT - $!"
 383                         if !open(STDOUT, ">&", fileno($outfh));
 384
 385                     exec($comp);
 386                     die "fork compressor '$comp' failed\n";
 387                 };
 388                 if (my $err = $@) {
 389                     $self->logerr($err);
 390                     POSIX::_exit(1);
 391                 }
 392                 POSIX::_exit(0);
 393                 kill(-9, $$);
 394             } else {
 395                 POSIX::close($pipefd[0]);
 396                 $outfileno = $pipefd[1];
 397             }
 398         } else {
 399             $outfileno = fileno($outfh);
 400         }
 401
 402         my $add_fd_cb = sub {
 403             my ($vmid, $resp) = @_;
 404
 405             my $params = {
 406                 'backup-file' => "/dev/fdname/backup",
 407                 speed => $speed,
 408                 'config-file' => $conffile,
 409                 devlist => $devlist
 410             };
 411
 412             $params->{'firewall-file'} = $firewall if -e $firewall;
 413             $qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params);
 414         };
 415
 416         $qmpclient->queue_cmd($vmid, $add_fd_cb, 'getfd',
 417                               fd => $outfileno, fdname => "backup");
 418
 419         my $agent_running = 0;
 420
 421         if ($self->{vmlist}->{$vmid}->{agent} && $vm_is_running) {
 422             $agent_running = PVE::QemuServer::qga_check_running($vmid);
 423             $self->loginfo("skipping guest-agent 'fs-freeze', agent configured but not running?")
 424                 if !$agent_running;
 425         }
 426
 427         if ($agent_running){
 428             $self->loginfo("issuing guest-agent 'fs-freeze' command");
 429             eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
 430             if (my $err = $@) {
 431                 $self->logerr($err);
 432             }
 433         }
 434
 435         eval { $qmpclient->queue_execute() };
 436         my $qmperr = $@;
 437
 438         if ($agent_running){
 439             $self->loginfo("issuing guest-agent 'fs-thaw' command");
 440             eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
 441             if (my $err = $@) {
 442                 $self->logerr($err);
 443             }
 444         }
 445         die $qmperr if $qmperr;
 446         die $qmpclient->{errors}->{$vmid} if $qmpclient->{errors}->{$vmid};
 447
 448         if ($cpid) {
 449             POSIX::close($outfileno) == 0 ||
 450                 die "close output file handle failed\n";
 451         }
 452
 453         die "got no uuid for backup task\n" if !$uuid;
 454
 455         $self->loginfo("started backup task '$uuid'");
 456
 457         if ($resume_on_backup) {
 458             if (my $stoptime = $task->{vmstoptime}) {
 459                 my $delay = time() - $task->{vmstoptime};
 460                 $task->{vmstoptime} = undef; # avoid printing 'online after ..' twice
 461                 $self->loginfo("resuming VM again after $delay seconds");
 462             } else {
 463                 $self->loginfo("resuming VM again");
 464             }
 465             mon_cmd($vmid, 'cont');
 466         }
 467
 468         my $status;
 469         my $starttime = time ();
 470         my $last_per = -1;
 471         my $last_total = 0;
 472         my $last_zero = 0;
 473         my $last_transferred = 0;
 474         my $last_time = time();
 475         my $transferred;
 476
 477         while(1) {
 478             $status = mon_cmd($vmid, 'query-backup');
 479             my $total = $status->{total} || 0;
 480             $transferred = $status->{transferred} || 0;
 481             my $per = $total ? int(($transferred * 100)/$total) : 0;
 482             my $zero = $status->{'zero-bytes'} || 0;
 483             my $zero_per = $total ? int(($zero * 100)/$total) : 0;
 484
 485             die "got unexpected uuid\n" if !$status->{uuid} || ($status->{uuid} ne $uuid);
 486
 487             my $ctime = time();
 488             my $duration = $ctime - $starttime;
 489
 490             my $rbytes = $transferred - $last_transferred;
 491             my $wbytes = $rbytes - ($zero - $last_zero);
 492
 493             my $timediff = ($ctime - $last_time) || 1; # fixme
 494             my $mbps_read = ($rbytes > 0) ?
 495                 int(($rbytes/$timediff)/(1000*1000)) : 0;
 496             my $mbps_write = ($wbytes > 0) ?
 497                 int(($wbytes/$timediff)/(1000*1000)) : 0;
 498
 499             my $statusline = "status: $per% ($transferred/$total), " .
 500                 "sparse ${zero_per}% ($zero), duration $duration, " .
 501                 "read/write $mbps_read/$mbps_write MB/s";
 502             my $res = $status->{status} || 'unknown';
 503             if ($res ne 'active') {
 504                 $self->loginfo($statusline);
 505                 die(($status->{errmsg} || "unknown error") . "\n")
 506                     if $res eq 'error';
 507                 die "got unexpected status '$res'\n"
 508                     if $res ne 'done';
 509                 die "got wrong number of transfered bytes ($total != $transferred)\n"
 510                     if ($res eq 'done') && ($total != $transferred);
 511
 512                 last;
 513             }
 514             if ($per != $last_per && ($timediff > 2)) {
 515                 $self->loginfo($statusline);
 516                 $last_per = $per;
 517                 $last_total = $total if $total;
 518                 $last_zero = $zero if $zero;
 519                 $last_transferred = $transferred if $transferred;
 520                 $last_time = $ctime;
 521             }
 522             sleep(1);
 523         }
 524
 525         my $duration = time() - $starttime;
 526         if ($transferred && $duration) {
 527             my $mb = int($transferred/(1000*1000));
 528             my $mbps = int(($transferred/$duration)/(1000*1000));
 529             $self->loginfo("transferred $mb MB in $duration seconds ($mbps MB/s)");
 530         }
 531     };
 532     my $err = $@;
 533
 534     if ($err) {
 535         $self->logerr($err);
 536         $self->loginfo("aborting backup job");
 537         eval { mon_cmd($vmid, 'backup-cancel'); };
 538         if (my $err1 = $@) {
 539             $self->logerr($err1);
 540         }
 541     }
 542
 543     if ($stop_after_backup) {
 544         # stop if not running
 545         eval {
 546             my $resp = mon_cmd($vmid, 'query-status');
 547             my $status = $resp && $resp->{status} ?  $resp->{status} : 'unknown';
 548             if ($status eq 'prelaunch') {
 549                 $self->loginfo("stopping kvm after backup task");
 550                 PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, $skiplock);
 551             } else {
 552                 $self->loginfo("kvm status changed after backup ('$status')" .
 553                                " - keep VM running");
 554             }
 555         }
 556     }
 557
 558     if ($err) {
 559         if ($cpid) {
 560             kill(9, $cpid);
 561             waitpid($cpid, 0);
 562         }
 563         die $err;
 564     }
 565
 566     if ($cpid && (waitpid($cpid, 0) > 0)) {
 567         my $stat = $?;
 568         my $ec = $stat >> 8;
 569         my $signal = $stat & 127;
 570         if ($ec || $signal) {
 571             die "$comp failed - wrong exit status $ec" .
 572                 ($signal ? " (signal $signal)\n" : "\n");
 573         }
 574     }
 575 }
 576
 577 sub snapshot {
 578     my ($self, $task, $vmid) = @_;
 579
 580     # nothing to do
 581 }
 582
 583 sub cleanup {
 584     my ($self, $task, $vmid) = @_;
 585
 586     # nothing to do ?
 587 }
 588
 589 1;