PVE/VZDump/QemuServer.pm

   1 package PVE::VZDump::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Basename;
   7 use File::Path;
   8 use IO::File;
   9 use IPC::Open3;
  10
  11 use PVE::Cluster qw(cfs_read_file);
  12 use PVE::INotify;
  13 use PVE::IPCC;
  14 use PVE::JSONSchema;
  15 use PVE::Storage::Plugin;
  16 use PVE::Storage;
  17 use PVE::Tools;
  18 use PVE::VZDump;
  19
  20 use PVE::QemuServer;
  21
  22 use base qw (PVE::VZDump::Plugin);
  23
  24 sub new {
  25     my ($class, $vzdump) = @_;
  26
  27     PVE::VZDump::check_bin('qm');
  28
  29     my $self = bless { vzdump => $vzdump }, $class;
  30
  31     $self->{vmlist} = PVE::QemuServer::vzlist();
  32     $self->{storecfg} = PVE::Storage::config();
  33
  34     return $self;
  35 };
  36
  37
  38 sub type {
  39     return 'qemu';
  40 }
  41
  42 sub vmlist {
  43     my ($self) = @_;
  44
  45     return [ keys %{$self->{vmlist}} ];
  46 }
  47
  48 sub prepare {
  49     my ($self, $task, $vmid, $mode) = @_;
  50
  51     $task->{disks} = [];
  52
  53     my $conf = $self->{vmlist}->{$vmid} = PVE::QemuConfig->load_config($vmid);
  54
  55     $self->loginfo("VM Name: $conf->{name}")
  56         if defined($conf->{name});
  57
  58     $self->{vm_was_running} = 1;
  59     if (!PVE::QemuServer::check_running($vmid)) {
  60         $self->{vm_was_running} = 0;
  61     }
  62
  63     $task->{hostname} = $conf->{name};
  64
  65     my $hostname = PVE::INotify::nodename();
  66
  67     my $vollist = [];
  68     my $drivehash = {};
  69     PVE::QemuServer::foreach_drive($conf, sub {
  70         my ($ds, $drive) = @_;
  71
  72         return if PVE::QemuServer::drive_is_cdrom($drive);
  73
  74         my $volid = $drive->{file};
  75
  76         if (defined($drive->{backup}) && !$drive->{backup}) {
  77             $self->loginfo("exclude disk '$ds' '$volid' (backup=no)");
  78             return;
  79         } elsif ($drive->{iothread}) {
  80             if (!PVE::QemuServer::runs_at_least_qemu_version($vmid, 4, 0, 1)) {
  81                 die "disk '$ds' '$volid' (iothread=on) can't use backup feature with running QEMU " .
  82                     "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n";
  83             }
  84         } else {
  85             my $log = "include disk '$ds' '$volid'";
  86            if (defined $drive->{size}) {
  87                 my $readable_size = PVE::JSONSchema::format_size($drive->{size});
  88                 $log .= " $readable_size";
  89            }
  90             $self->loginfo($log);
  91         }
  92
  93         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
  94         push @$vollist, $volid if $storeid;
  95         $drivehash->{$ds} = $drive;
  96     });
  97
  98     PVE::Storage::activate_volumes($self->{storecfg}, $vollist);
  99
 100     foreach my $ds (sort keys %$drivehash) {
 101         my $drive = $drivehash->{$ds};
 102
 103         my $volid = $drive->{file};
 104
 105         my $path;
 106
 107         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 108         if ($storeid) {
 109             $path = PVE::Storage::path($self->{storecfg}, $volid);
 110         } else {
 111             $path = $volid;
 112         }
 113
 114         next if !$path;
 115
 116         my $format = undef;
 117         my $size = undef;
 118
 119         eval{
 120             ($size, $format) = PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5);
 121         };
 122         die "no such volume '$volid'\n" if $@;
 123
 124         my $diskinfo = { path => $path , volid => $volid, storeid => $storeid,
 125                          format => $format, virtdev => $ds, qmdevice => "drive-$ds" };
 126
 127         if (-b $path) {
 128             $diskinfo->{type} = 'block';
 129         } else {
 130             $diskinfo->{type} = 'file';
 131         }
 132
 133         push @{$task->{disks}}, $diskinfo;
 134     }
 135 }
 136
 137 sub vm_status {
 138     my ($self, $vmid) = @_;
 139
 140     my $running = PVE::QemuServer::check_running($vmid) ? 1 : 0;
 141
 142     return wantarray ? ($running, $running ? 'running' : 'stopped') : $running;
 143 }
 144
 145 sub lock_vm {
 146     my ($self, $vmid) = @_;
 147
 148     $self->cmd ("qm set $vmid --lock backup");
 149 }
 150
 151 sub unlock_vm {
 152     my ($self, $vmid) = @_;
 153
 154     $self->cmd ("qm unlock $vmid");
 155 }
 156
 157 sub stop_vm {
 158     my ($self, $task, $vmid) = @_;
 159
 160     my $opts = $self->{vzdump}->{opts};
 161
 162     my $wait = $opts->{stopwait} * 60;
 163     # send shutdown and wait
 164     $self->cmd ("qm shutdown $vmid --skiplock --keepActive --timeout $wait");
 165 }
 166
 167 sub start_vm {
 168     my ($self, $task, $vmid) = @_;
 169
 170     $self->cmd ("qm start $vmid --skiplock");
 171 }
 172
 173 sub suspend_vm {
 174     my ($self, $task, $vmid) = @_;
 175
 176     $self->cmd ("qm suspend $vmid --skiplock");
 177 }
 178
 179 sub resume_vm {
 180     my ($self, $task, $vmid) = @_;
 181
 182     $self->cmd ("qm resume $vmid --skiplock");
 183 }
 184
 185 sub assemble {
 186     my ($self, $task, $vmid) = @_;
 187
 188     my $conffile = PVE::QemuConfig->config_file($vmid);
 189
 190     my $outfile = "$task->{tmpdir}/qemu-server.conf";
 191     my $firewall_src = "/etc/pve/firewall/$vmid.fw";
 192     my $firewall_dest = "$task->{tmpdir}/qemu-server.fw";
 193
 194     my $outfd = IO::File->new (">$outfile") ||
 195         die "unable to open '$outfile'";
 196     my $conffd = IO::File->new ($conffile, 'r') ||
 197         die "unable open '$conffile'";
 198
 199     my $found_snapshot;
 200     my $found_pending;
 201     while (defined (my $line = <$conffd>)) {
 202         next if $line =~ m/^\#vzdump\#/; # just to be sure
 203         next if $line =~ m/^\#qmdump\#/; # just to be sure
 204         if ($line =~ m/^\[(.*)\]\s*$/) {
 205             if ($1 =~ m/PENDING/i) {
 206                 $found_pending = 1;
 207             } else {
 208                 $found_snapshot = 1;
 209             }
 210         }
 211
 212         next if $found_snapshot; # skip all snapshots data
 213         next if $found_pending; # skip all pending changes
 214
 215         if ($line =~ m/^unused\d+:\s*(\S+)\s*/) {
 216             $self->loginfo("skip unused drive '$1' (not included into backup)");
 217             next;
 218         }
 219         next if $line =~ m/^lock:/ || $line =~ m/^parent:/;
 220
 221         print $outfd $line;
 222     }
 223
 224     foreach my $di (@{$task->{disks}}) {
 225         if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 226             my $storeid = $di->{storeid} || '';
 227             my $format = $di->{format} || '';
 228             print $outfd "#qmdump#map:$di->{virtdev}:$di->{qmdevice}:$storeid:$format:\n";
 229         } else {
 230             die "internal error";
 231         }
 232     }
 233
 234     if ($found_snapshot) {
 235         $self->loginfo("snapshots found (not included into backup)");
 236     }
 237
 238     if ($found_pending) {
 239         $self->loginfo("pending configuration changes found (not included into backup)");
 240     }
 241
 242     PVE::Tools::file_copy($firewall_src, $firewall_dest) if -f $firewall_src;
 243 }
 244
 245 sub archive {
 246     my ($self, $task, $vmid, $filename, $comp) = @_;
 247
 248     my $conffile = "$task->{tmpdir}/qemu-server.conf";
 249     my $firewall = "$task->{tmpdir}/qemu-server.fw";
 250
 251     my $opts = $self->{vzdump}->{opts};
 252
 253     my $starttime = time ();
 254
 255     my $speed = 0;
 256     if ($opts->{bwlimit}) {
 257         $speed = $opts->{bwlimit}*1024;
 258     }
 259
 260     my $diskcount = scalar(@{$task->{disks}});
 261
 262     if (PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}) || !$diskcount) {
 263         my @pathlist;
 264         foreach my $di (@{$task->{disks}}) {
 265             if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 266                 push @pathlist, "$di->{qmdevice}=$di->{path}";
 267             } else {
 268                 die "implement me";
 269             }
 270         }
 271
 272         if (!$diskcount) {
 273             $self->loginfo("backup contains no disks");
 274         }
 275
 276         my $outcmd;
 277         if ($comp) {
 278             $outcmd = "exec:$comp";
 279         } else {
 280             $outcmd = "exec:cat";
 281         }
 282
 283         $outcmd .= " > $filename" if !$opts->{stdout};
 284
 285         my $cmd = ['/usr/bin/vma', 'create', '-v', '-c', $conffile];
 286         push @$cmd, '-c', $firewall if -e $firewall;
 287         push @$cmd, $outcmd, @pathlist;
 288
 289         $self->loginfo("starting template backup");
 290         $self->loginfo(join(' ', @$cmd));
 291
 292         if ($opts->{stdout}) {
 293             $self->cmd($cmd, output => ">&=" . fileno($opts->{stdout}));
 294         } else {
 295             $self->cmd($cmd);
 296         }
 297
 298         return;
 299     }
 300
 301
 302     my $devlist = '';
 303     foreach my $di (@{$task->{disks}}) {
 304         if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 305             $devlist .= $devlist ? ",$di->{qmdevice}" : $di->{qmdevice};
 306         } else {
 307             die "implement me";
 308         }
 309     }
 310
 311     my $stop_after_backup;
 312     my $resume_on_backup;
 313
 314     my $skiplock = 1;
 315     my $vm_is_running = PVE::QemuServer::check_running($vmid);
 316     if (!$vm_is_running) {
 317         eval {
 318             $self->loginfo("starting kvm to execute backup task");
 319             PVE::QemuServer::vm_start($self->{storecfg}, $vmid, undef,
 320                                       $skiplock, undef, 1);
 321             if ($self->{vm_was_running}) {
 322                 $resume_on_backup = 1;
 323             } else {
 324                 $stop_after_backup = 1;
 325             }
 326         };
 327         if (my $err = $@) {
 328             die $err;
 329         }
 330     }
 331
 332     my $cpid;
 333     my $interrupt_msg = "interrupted by signal\n";
 334     eval {
 335         $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
 336             die $interrupt_msg;
 337         };
 338
 339         my $qmpclient = PVE::QMPClient->new();
 340
 341         my $uuid;
 342
 343         my $backup_cb = sub {
 344             my ($vmid, $resp) = @_;
 345             $uuid = $resp->{return}->{UUID};
 346         };
 347
 348         my $outfh;
 349         if ($opts->{stdout}) {
 350             $outfh = $opts->{stdout};
 351         } else {
 352             $outfh = IO::File->new($filename, "w") ||
 353                 die "unable to open file '$filename' - $!\n";
 354         }
 355
 356         my $outfileno;
 357         if ($comp) {
 358             my @pipefd = POSIX::pipe();
 359             $cpid = fork();
 360             die "unable to fork worker - $!" if !defined($cpid);
 361             if ($cpid == 0) {
 362                 eval {
 363                     POSIX::close($pipefd[1]);
 364                     # redirect STDIN
 365                     my $fd = fileno(STDIN);
 366                     close STDIN;
 367                     POSIX::close(0) if $fd != 0;
 368                     die "unable to redirect STDIN - $!"
 369                         if !open(STDIN, "<&", $pipefd[0]);
 370
 371                     # redirect STDOUT
 372                     $fd = fileno(STDOUT);
 373                     close STDOUT;
 374                     POSIX::close (1) if $fd != 1;
 375
 376                     die "unable to redirect STDOUT - $!"
 377                         if !open(STDOUT, ">&", fileno($outfh));
 378
 379                     exec($comp);
 380                     die "fork compressor '$comp' failed\n";
 381                 };
 382                 if (my $err = $@) {
 383                     $self->logerr($err);
 384                     POSIX::_exit(1);
 385                 }
 386                 POSIX::_exit(0);
 387                 kill(-9, $$);
 388             } else {
 389                 POSIX::close($pipefd[0]);
 390                 $outfileno = $pipefd[1];
 391             }
 392         } else {
 393             $outfileno = fileno($outfh);
 394         }
 395
 396         my $add_fd_cb = sub {
 397             my ($vmid, $resp) = @_;
 398
 399             my $params = {
 400                 'backup-file' => "/dev/fdname/backup",
 401                 speed => $speed,
 402                 'config-file' => $conffile,
 403                 devlist => $devlist
 404             };
 405
 406             $params->{'firewall-file'} = $firewall if -e $firewall;
 407             $qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params);
 408         };
 409
 410         $qmpclient->queue_cmd($vmid, $add_fd_cb, 'getfd',
 411                               fd => $outfileno, fdname => "backup");
 412
 413         my $agent_running = 0;
 414
 415         if ($self->{vmlist}->{$vmid}->{agent} && $vm_is_running) {
 416             $agent_running = PVE::QemuServer::qga_check_running($vmid);
 417         }
 418
 419         if ($agent_running){
 420             eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
 421             if (my $err = $@) {
 422                 $self->logerr($err);
 423             }
 424         }
 425
 426         eval { $qmpclient->queue_execute() };
 427         my $qmperr = $@;
 428
 429         if ($agent_running){
 430             eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
 431             if (my $err = $@) {
 432                 $self->logerr($err);
 433             }
 434         }
 435         die $qmperr if $qmperr;
 436         die $qmpclient->{errors}->{$vmid} if $qmpclient->{errors}->{$vmid};
 437
 438         if ($cpid) {
 439             POSIX::close($outfileno) == 0 ||
 440                 die "close output file handle failed\n";
 441         }
 442
 443         die "got no uuid for backup task\n" if !$uuid;
 444
 445         $self->loginfo("started backup task '$uuid'");
 446
 447         if ($resume_on_backup) {
 448             if (my $stoptime = $task->{vmstoptime}) {
 449                 my $delay = time() - $task->{vmstoptime};
 450                 $task->{vmstoptime} = undef; # avoid printing 'online after ..' twice
 451                 $self->loginfo("resuming VM again after $delay seconds");
 452             } else {
 453                 $self->loginfo("resuming VM again");
 454             }
 455             PVE::QemuServer::vm_mon_cmd($vmid, 'cont');
 456         }
 457
 458         my $status;
 459         my $starttime = time ();
 460         my $last_per = -1;
 461         my $last_total = 0;
 462         my $last_zero = 0;
 463         my $last_transferred = 0;
 464         my $last_time = time();
 465         my $transferred;
 466
 467         while(1) {
 468             $status = PVE::QemuServer::vm_mon_cmd($vmid, 'query-backup');
 469             my $total = $status->{total} || 0;
 470             $transferred = $status->{transferred} || 0;
 471             my $per = $total ? int(($transferred * 100)/$total) : 0;
 472             my $zero = $status->{'zero-bytes'} || 0;
 473             my $zero_per = $total ? int(($zero * 100)/$total) : 0;
 474
 475             die "got unexpected uuid\n" if !$status->{uuid} || ($status->{uuid} ne $uuid);
 476
 477             my $ctime = time();
 478             my $duration = $ctime - $starttime;
 479
 480             my $rbytes = $transferred - $last_transferred;
 481             my $wbytes = $rbytes - ($zero - $last_zero);
 482
 483             my $timediff = ($ctime - $last_time) || 1; # fixme
 484             my $mbps_read = ($rbytes > 0) ?
 485                 int(($rbytes/$timediff)/(1000*1000)) : 0;
 486             my $mbps_write = ($wbytes > 0) ?
 487                 int(($wbytes/$timediff)/(1000*1000)) : 0;
 488
 489             my $statusline = "status: $per% ($transferred/$total), " .
 490                 "sparse ${zero_per}% ($zero), duration $duration, " .
 491                 "read/write $mbps_read/$mbps_write MB/s";
 492             my $res = $status->{status} || 'unknown';
 493             if ($res ne 'active') {
 494                 $self->loginfo($statusline);
 495                 die(($status->{errmsg} || "unknown error") . "\n")
 496                     if $res eq 'error';
 497                 die "got unexpected status '$res'\n"
 498                     if $res ne 'done';
 499                 die "got wrong number of transfered bytes ($total != $transferred)\n"
 500                     if ($res eq 'done') && ($total != $transferred);
 501
 502                 last;
 503             }
 504             if ($per != $last_per && ($timediff > 2)) {
 505                 $self->loginfo($statusline);
 506                 $last_per = $per;
 507                 $last_total = $total if $total;
 508                 $last_zero = $zero if $zero;
 509                 $last_transferred = $transferred if $transferred;
 510                 $last_time = $ctime;
 511             }
 512             sleep(1);
 513         }
 514
 515         my $duration = time() - $starttime;
 516         if ($transferred && $duration) {
 517             my $mb = int($transferred/(1000*1000));
 518             my $mbps = int(($transferred/$duration)/(1000*1000));
 519             $self->loginfo("transferred $mb MB in $duration seconds ($mbps MB/s)");
 520         }
 521     };
 522     my $err = $@;
 523
 524     if ($err) {
 525         $self->logerr($err);
 526         $self->loginfo("aborting backup job");
 527         eval { PVE::QemuServer::vm_mon_cmd($vmid, 'backup-cancel'); };
 528         if (my $err1 = $@) {
 529             $self->logerr($err1);
 530         }
 531     }
 532
 533     if ($stop_after_backup) {
 534         # stop if not running
 535         eval {
 536             my $resp = PVE::QemuServer::vm_mon_cmd($vmid, 'query-status');
 537             my $status = $resp && $resp->{status} ?  $resp->{status} : 'unknown';
 538             if ($status eq 'prelaunch') {
 539                 $self->loginfo("stopping kvm after backup task");
 540                 PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, $skiplock);
 541             } else {
 542                 $self->loginfo("kvm status changed after backup ('$status')" .
 543                                " - keep VM running");
 544             }
 545         }
 546     }
 547
 548     if ($err) {
 549         if ($cpid) {
 550             kill(9, $cpid);
 551             waitpid($cpid, 0);
 552         }
 553         die $err;
 554     }
 555
 556     if ($cpid && (waitpid($cpid, 0) > 0)) {
 557         my $stat = $?;
 558         my $ec = $stat >> 8;
 559         my $signal = $stat & 127;
 560         if ($ec || $signal) {
 561             die "$comp failed - wrong exit status $ec" .
 562                 ($signal ? " (signal $signal)\n" : "\n");
 563         }
 564     }
 565 }
 566
 567 sub snapshot {
 568     my ($self, $task, $vmid) = @_;
 569
 570     # nothing to do
 571 }
 572
 573 sub cleanup {
 574     my ($self, $task, $vmid) = @_;
 575
 576     # nothing to do ?
 577 }
 578
 579 1;