PVE/VZDump/QemuServer.pm

   1 package PVE::VZDump::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Basename;
   7 use File::Path;
   8 use IO::File;
   9 use IPC::Open3;
  10
  11 use PVE::Cluster qw(cfs_read_file);
  12 use PVE::INotify;
  13 use PVE::IPCC;
  14 use PVE::JSONSchema;
  15 use PVE::QMPClient;
  16 use PVE::Storage::Plugin;
  17 use PVE::Storage;
  18 use PVE::Tools;
  19 use PVE::VZDump;
  20
  21 use PVE::QemuServer;
  22 use PVE::QemuServer::Monitor qw(mon_cmd);
  23
  24 use base qw (PVE::VZDump::Plugin);
  25
  26 sub new {
  27     my ($class, $vzdump) = @_;
  28
  29     PVE::VZDump::check_bin('qm');
  30
  31     my $self = bless { vzdump => $vzdump }, $class;
  32
  33     $self->{vmlist} = PVE::QemuServer::vzlist();
  34     $self->{storecfg} = PVE::Storage::config();
  35
  36     return $self;
  37 };
  38
  39
  40 sub type {
  41     return 'qemu';
  42 }
  43
  44 sub vmlist {
  45     my ($self) = @_;
  46
  47     return [ keys %{$self->{vmlist}} ];
  48 }
  49
  50 sub prepare {
  51     my ($self, $task, $vmid, $mode) = @_;
  52
  53     $task->{disks} = [];
  54
  55     my $conf = $self->{vmlist}->{$vmid} = PVE::QemuConfig->load_config($vmid);
  56
  57     $self->loginfo("VM Name: $conf->{name}")
  58         if defined($conf->{name});
  59
  60     $self->{vm_was_running} = 1;
  61     if (!PVE::QemuServer::check_running($vmid)) {
  62         $self->{vm_was_running} = 0;
  63     }
  64
  65     $task->{hostname} = $conf->{name};
  66
  67     my $hostname = PVE::INotify::nodename();
  68
  69     my $vollist = [];
  70     my $drivehash = {};
  71     PVE::QemuServer::foreach_drive($conf, sub {
  72         my ($ds, $drive) = @_;
  73
  74         return if PVE::QemuServer::drive_is_cdrom($drive);
  75
  76         my $volid = $drive->{file};
  77
  78         if (defined($drive->{backup}) && !$drive->{backup}) {
  79             $self->loginfo("exclude disk '$ds' '$volid' (backup=no)");
  80             return;
  81         } elsif ($self->{vm_was_running} && $drive->{iothread}) {
  82             if (!PVE::QemuServer::runs_at_least_qemu_version($vmid, 4, 0, 1)) {
  83                 die "disk '$ds' '$volid' (iothread=on) can't use backup feature with running QEMU " .
  84                     "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n";
  85             }
  86         } else {
  87             my $log = "include disk '$ds' '$volid'";
  88            if (defined $drive->{size}) {
  89                 my $readable_size = PVE::JSONSchema::format_size($drive->{size});
  90                 $log .= " $readable_size";
  91            }
  92             $self->loginfo($log);
  93         }
  94
  95         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
  96         push @$vollist, $volid if $storeid;
  97         $drivehash->{$ds} = $drive;
  98     });
  99
 100     PVE::Storage::activate_volumes($self->{storecfg}, $vollist);
 101
 102     foreach my $ds (sort keys %$drivehash) {
 103         my $drive = $drivehash->{$ds};
 104
 105         my $volid = $drive->{file};
 106
 107         my $path;
 108
 109         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 110         if ($storeid) {
 111             $path = PVE::Storage::path($self->{storecfg}, $volid);
 112         } else {
 113             $path = $volid;
 114         }
 115
 116         next if !$path;
 117
 118         my $format = undef;
 119         my $size = undef;
 120
 121         eval{
 122             ($size, $format) = PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5);
 123         };
 124         die "no such volume '$volid'\n" if $@;
 125
 126         my $diskinfo = { path => $path , volid => $volid, storeid => $storeid,
 127                          format => $format, virtdev => $ds, qmdevice => "drive-$ds" };
 128
 129         if (-b $path) {
 130             $diskinfo->{type} = 'block';
 131         } else {
 132             $diskinfo->{type} = 'file';
 133         }
 134
 135         push @{$task->{disks}}, $diskinfo;
 136     }
 137 }
 138
 139 sub vm_status {
 140     my ($self, $vmid) = @_;
 141
 142     my $running = PVE::QemuServer::check_running($vmid) ? 1 : 0;
 143
 144     return wantarray ? ($running, $running ? 'running' : 'stopped') : $running;
 145 }
 146
 147 sub lock_vm {
 148     my ($self, $vmid) = @_;
 149
 150     $self->cmd ("qm set $vmid --lock backup");
 151 }
 152
 153 sub unlock_vm {
 154     my ($self, $vmid) = @_;
 155
 156     $self->cmd ("qm unlock $vmid");
 157 }
 158
 159 sub stop_vm {
 160     my ($self, $task, $vmid) = @_;
 161
 162     my $opts = $self->{vzdump}->{opts};
 163
 164     my $wait = $opts->{stopwait} * 60;
 165     # send shutdown and wait
 166     $self->cmd ("qm shutdown $vmid --skiplock --keepActive --timeout $wait");
 167 }
 168
 169 sub start_vm {
 170     my ($self, $task, $vmid) = @_;
 171
 172     $self->cmd ("qm start $vmid --skiplock");
 173 }
 174
 175 sub suspend_vm {
 176     my ($self, $task, $vmid) = @_;
 177
 178     $self->cmd ("qm suspend $vmid --skiplock");
 179 }
 180
 181 sub resume_vm {
 182     my ($self, $task, $vmid) = @_;
 183
 184     $self->cmd ("qm resume $vmid --skiplock");
 185 }
 186
 187 sub assemble {
 188     my ($self, $task, $vmid) = @_;
 189
 190     my $conffile = PVE::QemuConfig->config_file($vmid);
 191
 192     my $outfile = "$task->{tmpdir}/qemu-server.conf";
 193     my $firewall_src = "/etc/pve/firewall/$vmid.fw";
 194     my $firewall_dest = "$task->{tmpdir}/qemu-server.fw";
 195
 196     my $outfd = IO::File->new (">$outfile") ||
 197         die "unable to open '$outfile'";
 198     my $conffd = IO::File->new ($conffile, 'r') ||
 199         die "unable open '$conffile'";
 200
 201     my $found_snapshot;
 202     my $found_pending;
 203     while (defined (my $line = <$conffd>)) {
 204         next if $line =~ m/^\#vzdump\#/; # just to be sure
 205         next if $line =~ m/^\#qmdump\#/; # just to be sure
 206         if ($line =~ m/^\[(.*)\]\s*$/) {
 207             if ($1 =~ m/PENDING/i) {
 208                 $found_pending = 1;
 209             } else {
 210                 $found_snapshot = 1;
 211             }
 212         }
 213
 214         next if $found_snapshot; # skip all snapshots data
 215         next if $found_pending; # skip all pending changes
 216
 217         if ($line =~ m/^unused\d+:\s*(\S+)\s*/) {
 218             $self->loginfo("skip unused drive '$1' (not included into backup)");
 219             next;
 220         }
 221         next if $line =~ m/^lock:/ || $line =~ m/^parent:/;
 222
 223         print $outfd $line;
 224     }
 225
 226     foreach my $di (@{$task->{disks}}) {
 227         if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 228             my $storeid = $di->{storeid} || '';
 229             my $format = $di->{format} || '';
 230             print $outfd "#qmdump#map:$di->{virtdev}:$di->{qmdevice}:$storeid:$format:\n";
 231         } else {
 232             die "internal error";
 233         }
 234     }
 235
 236     if ($found_snapshot) {
 237         $self->loginfo("snapshots found (not included into backup)");
 238     }
 239
 240     if ($found_pending) {
 241         $self->loginfo("pending configuration changes found (not included into backup)");
 242     }
 243
 244     PVE::Tools::file_copy($firewall_src, $firewall_dest) if -f $firewall_src;
 245 }
 246
 247 sub archive {
 248     my ($self, $task, $vmid, $filename, $comp) = @_;
 249
 250     my $conffile = "$task->{tmpdir}/qemu-server.conf";
 251     my $firewall = "$task->{tmpdir}/qemu-server.fw";
 252
 253     my $opts = $self->{vzdump}->{opts};
 254
 255     my $starttime = time ();
 256
 257     my $speed = 0;
 258     if ($opts->{bwlimit}) {
 259         $speed = $opts->{bwlimit}*1024;
 260     }
 261
 262     my $diskcount = scalar(@{$task->{disks}});
 263
 264     if (PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}) || !$diskcount) {
 265         my @pathlist;
 266         foreach my $di (@{$task->{disks}}) {
 267             if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 268                 push @pathlist, "$di->{qmdevice}=$di->{path}";
 269             } else {
 270                 die "implement me";
 271             }
 272         }
 273
 274         if (!$diskcount) {
 275             $self->loginfo("backup contains no disks");
 276         }
 277
 278         my $outcmd;
 279         if ($comp) {
 280             $outcmd = "exec:$comp";
 281         } else {
 282             $outcmd = "exec:cat";
 283         }
 284
 285         $outcmd .= " > $filename" if !$opts->{stdout};
 286
 287         my $cmd = ['/usr/bin/vma', 'create', '-v', '-c', $conffile];
 288         push @$cmd, '-c', $firewall if -e $firewall;
 289         push @$cmd, $outcmd, @pathlist;
 290
 291         $self->loginfo("starting template backup");
 292         $self->loginfo(join(' ', @$cmd));
 293
 294         if ($opts->{stdout}) {
 295             $self->cmd($cmd, output => ">&=" . fileno($opts->{stdout}));
 296         } else {
 297             $self->cmd($cmd);
 298         }
 299
 300         return;
 301     }
 302
 303
 304     my $devlist = '';
 305     foreach my $di (@{$task->{disks}}) {
 306         if ($di->{type} eq 'block' || $di->{type} eq 'file') {
 307             $devlist .= $devlist ? ",$di->{qmdevice}" : $di->{qmdevice};
 308         } else {
 309             die "implement me";
 310         }
 311     }
 312
 313     my $stop_after_backup;
 314     my $resume_on_backup;
 315
 316     my $skiplock = 1;
 317     my $vm_is_running = PVE::QemuServer::check_running($vmid);
 318     if (!$vm_is_running) {
 319         eval {
 320             $self->loginfo("starting kvm to execute backup task");
 321             PVE::QemuServer::vm_start($self->{storecfg}, $vmid, undef,
 322                                       $skiplock, undef, 1);
 323             if ($self->{vm_was_running}) {
 324                 $resume_on_backup = 1;
 325             } else {
 326                 $stop_after_backup = 1;
 327             }
 328         };
 329         if (my $err = $@) {
 330             die $err;
 331         }
 332     }
 333
 334     my $cpid;
 335     my $interrupt_msg = "interrupted by signal\n";
 336     eval {
 337         $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
 338             die $interrupt_msg;
 339         };
 340
 341         my $qmpclient = PVE::QMPClient->new();
 342
 343         my $uuid;
 344
 345         my $backup_cb = sub {
 346             my ($vmid, $resp) = @_;
 347             $uuid = $resp->{return}->{UUID};
 348         };
 349
 350         my $outfh;
 351         if ($opts->{stdout}) {
 352             $outfh = $opts->{stdout};
 353         } else {
 354             $outfh = IO::File->new($filename, "w") ||
 355                 die "unable to open file '$filename' - $!\n";
 356         }
 357
 358         my $outfileno;
 359         if ($comp) {
 360             my @pipefd = POSIX::pipe();
 361             $cpid = fork();
 362             die "unable to fork worker - $!" if !defined($cpid);
 363             if ($cpid == 0) {
 364                 eval {
 365                     POSIX::close($pipefd[1]);
 366                     # redirect STDIN
 367                     my $fd = fileno(STDIN);
 368                     close STDIN;
 369                     POSIX::close(0) if $fd != 0;
 370                     die "unable to redirect STDIN - $!"
 371                         if !open(STDIN, "<&", $pipefd[0]);
 372
 373                     # redirect STDOUT
 374                     $fd = fileno(STDOUT);
 375                     close STDOUT;
 376                     POSIX::close (1) if $fd != 1;
 377
 378                     die "unable to redirect STDOUT - $!"
 379                         if !open(STDOUT, ">&", fileno($outfh));
 380
 381                     exec($comp);
 382                     die "fork compressor '$comp' failed\n";
 383                 };
 384                 if (my $err = $@) {
 385                     $self->logerr($err);
 386                     POSIX::_exit(1);
 387                 }
 388                 POSIX::_exit(0);
 389                 kill(-9, $$);
 390             } else {
 391                 POSIX::close($pipefd[0]);
 392                 $outfileno = $pipefd[1];
 393             }
 394         } else {
 395             $outfileno = fileno($outfh);
 396         }
 397
 398         my $add_fd_cb = sub {
 399             my ($vmid, $resp) = @_;
 400
 401             my $params = {
 402                 'backup-file' => "/dev/fdname/backup",
 403                 speed => $speed,
 404                 'config-file' => $conffile,
 405                 devlist => $devlist
 406             };
 407
 408             $params->{'firewall-file'} = $firewall if -e $firewall;
 409             $qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params);
 410         };
 411
 412         $qmpclient->queue_cmd($vmid, $add_fd_cb, 'getfd',
 413                               fd => $outfileno, fdname => "backup");
 414
 415         my $agent_running = 0;
 416
 417         if ($self->{vmlist}->{$vmid}->{agent} && $vm_is_running) {
 418             $agent_running = PVE::QemuServer::qga_check_running($vmid);
 419         }
 420
 421         if ($agent_running){
 422             eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
 423             if (my $err = $@) {
 424                 $self->logerr($err);
 425             }
 426         }
 427
 428         eval { $qmpclient->queue_execute() };
 429         my $qmperr = $@;
 430
 431         if ($agent_running){
 432             eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
 433             if (my $err = $@) {
 434                 $self->logerr($err);
 435             }
 436         }
 437         die $qmperr if $qmperr;
 438         die $qmpclient->{errors}->{$vmid} if $qmpclient->{errors}->{$vmid};
 439
 440         if ($cpid) {
 441             POSIX::close($outfileno) == 0 ||
 442                 die "close output file handle failed\n";
 443         }
 444
 445         die "got no uuid for backup task\n" if !$uuid;
 446
 447         $self->loginfo("started backup task '$uuid'");
 448
 449         if ($resume_on_backup) {
 450             if (my $stoptime = $task->{vmstoptime}) {
 451                 my $delay = time() - $task->{vmstoptime};
 452                 $task->{vmstoptime} = undef; # avoid printing 'online after ..' twice
 453                 $self->loginfo("resuming VM again after $delay seconds");
 454             } else {
 455                 $self->loginfo("resuming VM again");
 456             }
 457             mon_cmd($vmid, 'cont');
 458         }
 459
 460         my $status;
 461         my $starttime = time ();
 462         my $last_per = -1;
 463         my $last_total = 0;
 464         my $last_zero = 0;
 465         my $last_transferred = 0;
 466         my $last_time = time();
 467         my $transferred;
 468
 469         while(1) {
 470             $status = mon_cmd($vmid, 'query-backup');
 471             my $total = $status->{total} || 0;
 472             $transferred = $status->{transferred} || 0;
 473             my $per = $total ? int(($transferred * 100)/$total) : 0;
 474             my $zero = $status->{'zero-bytes'} || 0;
 475             my $zero_per = $total ? int(($zero * 100)/$total) : 0;
 476
 477             die "got unexpected uuid\n" if !$status->{uuid} || ($status->{uuid} ne $uuid);
 478
 479             my $ctime = time();
 480             my $duration = $ctime - $starttime;
 481
 482             my $rbytes = $transferred - $last_transferred;
 483             my $wbytes = $rbytes - ($zero - $last_zero);
 484
 485             my $timediff = ($ctime - $last_time) || 1; # fixme
 486             my $mbps_read = ($rbytes > 0) ?
 487                 int(($rbytes/$timediff)/(1000*1000)) : 0;
 488             my $mbps_write = ($wbytes > 0) ?
 489                 int(($wbytes/$timediff)/(1000*1000)) : 0;
 490
 491             my $statusline = "status: $per% ($transferred/$total), " .
 492                 "sparse ${zero_per}% ($zero), duration $duration, " .
 493                 "read/write $mbps_read/$mbps_write MB/s";
 494             my $res = $status->{status} || 'unknown';
 495             if ($res ne 'active') {
 496                 $self->loginfo($statusline);
 497                 die(($status->{errmsg} || "unknown error") . "\n")
 498                     if $res eq 'error';
 499                 die "got unexpected status '$res'\n"
 500                     if $res ne 'done';
 501                 die "got wrong number of transfered bytes ($total != $transferred)\n"
 502                     if ($res eq 'done') && ($total != $transferred);
 503
 504                 last;
 505             }
 506             if ($per != $last_per && ($timediff > 2)) {
 507                 $self->loginfo($statusline);
 508                 $last_per = $per;
 509                 $last_total = $total if $total;
 510                 $last_zero = $zero if $zero;
 511                 $last_transferred = $transferred if $transferred;
 512                 $last_time = $ctime;
 513             }
 514             sleep(1);
 515         }
 516
 517         my $duration = time() - $starttime;
 518         if ($transferred && $duration) {
 519             my $mb = int($transferred/(1000*1000));
 520             my $mbps = int(($transferred/$duration)/(1000*1000));
 521             $self->loginfo("transferred $mb MB in $duration seconds ($mbps MB/s)");
 522         }
 523     };
 524     my $err = $@;
 525
 526     if ($err) {
 527         $self->logerr($err);
 528         $self->loginfo("aborting backup job");
 529         eval { mon_cmd($vmid, 'backup-cancel'); };
 530         if (my $err1 = $@) {
 531             $self->logerr($err1);
 532         }
 533     }
 534
 535     if ($stop_after_backup) {
 536         # stop if not running
 537         eval {
 538             my $resp = mon_cmd($vmid, 'query-status');
 539             my $status = $resp && $resp->{status} ?  $resp->{status} : 'unknown';
 540             if ($status eq 'prelaunch') {
 541                 $self->loginfo("stopping kvm after backup task");
 542                 PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, $skiplock);
 543             } else {
 544                 $self->loginfo("kvm status changed after backup ('$status')" .
 545                                " - keep VM running");
 546             }
 547         }
 548     }
 549
 550     if ($err) {
 551         if ($cpid) {
 552             kill(9, $cpid);
 553             waitpid($cpid, 0);
 554         }
 555         die $err;
 556     }
 557
 558     if ($cpid && (waitpid($cpid, 0) > 0)) {
 559         my $stat = $?;
 560         my $ec = $stat >> 8;
 561         my $signal = $stat & 127;
 562         if ($ec || $signal) {
 563             die "$comp failed - wrong exit status $ec" .
 564                 ($signal ? " (signal $signal)\n" : "\n");
 565         }
 566     }
 567 }
 568
 569 sub snapshot {
 570     my ($self, $task, $vmid) = @_;
 571
 572     # nothing to do
 573 }
 574
 575 sub cleanup {
 576     my ($self, $task, $vmid) = @_;
 577
 578     # nothing to do ?
 579 }
 580
 581 1;