PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  35
  36 # Note about locking: we use flock on the config file protect
  37 # against concurent actions.
  38 # Aditionaly, we have a 'lock' setting in the config file. This
  39 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  40 # allowed when such lock is set. But you can ignore this kind of
  41 # lock with the --skiplock flag.
  42
  43 cfs_register_file('/qemu-server/',
  44                   \&parse_vm_config,
  45                   \&write_vm_config);
  46
  47 PVE::JSONSchema::register_standard_option('skiplock', {
  48     description => "Ignore locks - only root is allowed to use this option.",
  49     type => 'boolean',
  50     optional => 1,
  51 });
  52
  53 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  54     description => "Some command save/restore state from this location.",
  55     type => 'string',
  56     maxLength => 128,
  57     optional => 1,
  58 });
  59
  60 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  61     description => "The name of the snapshot.",
  62     type => 'string', format => 'pve-configid',
  63     maxLength => 40,
  64 });
  65
  66 #no warnings 'redefine';
  67
  68 unless(defined(&_VZSYSCALLS_H_)) {
  69     eval 'sub _VZSYSCALLS_H_ () {1;}' unless defined(&_VZSYSCALLS_H_);
  70     require 'sys/syscall.ph';
  71     if(defined(&__x86_64__)) {
  72         eval 'sub __NR_fairsched_vcpus () {499;}' unless defined(&__NR_fairsched_vcpus);
  73         eval 'sub __NR_fairsched_mknod () {504;}' unless defined(&__NR_fairsched_mknod);
  74         eval 'sub __NR_fairsched_rmnod () {505;}' unless defined(&__NR_fairsched_rmnod);
  75         eval 'sub __NR_fairsched_chwt () {506;}' unless defined(&__NR_fairsched_chwt);
  76         eval 'sub __NR_fairsched_mvpr () {507;}' unless defined(&__NR_fairsched_mvpr);
  77         eval 'sub __NR_fairsched_rate () {508;}' unless defined(&__NR_fairsched_rate);
  78         eval 'sub __NR_setluid () {501;}' unless defined(&__NR_setluid);
  79         eval 'sub __NR_setublimit () {502;}' unless defined(&__NR_setublimit);
  80     }
  81     elsif(defined( &__i386__) ) {
  82         eval 'sub __NR_fairsched_mknod () {500;}' unless defined(&__NR_fairsched_mknod);
  83         eval 'sub __NR_fairsched_rmnod () {501;}' unless defined(&__NR_fairsched_rmnod);
  84         eval 'sub __NR_fairsched_chwt () {502;}' unless defined(&__NR_fairsched_chwt);
  85         eval 'sub __NR_fairsched_mvpr () {503;}' unless defined(&__NR_fairsched_mvpr);
  86         eval 'sub __NR_fairsched_rate () {504;}' unless defined(&__NR_fairsched_rate);
  87         eval 'sub __NR_fairsched_vcpus () {505;}' unless defined(&__NR_fairsched_vcpus);
  88         eval 'sub __NR_setluid () {511;}' unless defined(&__NR_setluid);
  89         eval 'sub __NR_setublimit () {512;}' unless defined(&__NR_setublimit);
  90     } else {
  91         die("no fairsched syscall for this arch");
  92     }
  93     require 'asm/ioctl.ph';
  94     eval 'sub KVM_GET_API_VERSION () { &_IO(0xAE, 0x);}' unless defined(&KVM_GET_API_VERSION);
  95 }
  96
  97 sub fairsched_mknod {
  98     my ($parent, $weight, $desired) = @_;
  99
 100     return syscall(&__NR_fairsched_mknod, int($parent), int($weight), int($desired));
 101 }
 102
 103 sub fairsched_rmnod {
 104     my ($id) = @_;
 105
 106     return syscall(&__NR_fairsched_rmnod, int($id));
 107 }
 108
 109 sub fairsched_mvpr {
 110     my ($pid, $newid) = @_;
 111
 112     return syscall(&__NR_fairsched_mvpr, int($pid), int($newid));
 113 }
 114
 115 sub fairsched_vcpus {
 116     my ($id, $vcpus) = @_;
 117
 118     return syscall(&__NR_fairsched_vcpus, int($id), int($vcpus));
 119 }
 120
 121 sub fairsched_rate {
 122     my ($id, $op, $rate) = @_;
 123
 124     return syscall(&__NR_fairsched_rate, int($id), int($op), int($rate));
 125 }
 126
 127 use constant FAIRSCHED_SET_RATE  => 0;
 128 use constant FAIRSCHED_DROP_RATE => 1;
 129 use constant FAIRSCHED_GET_RATE  => 2;
 130
 131 sub fairsched_cpulimit {
 132     my ($id, $limit) = @_;
 133
 134     my $cpulim1024 = int($limit * 1024 / 100);
 135     my $op = $cpulim1024 ? FAIRSCHED_SET_RATE : FAIRSCHED_DROP_RATE;
 136
 137     return fairsched_rate($id, $op, $cpulim1024);
 138 }
 139
 140 my $nodename = PVE::INotify::nodename();
 141
 142 mkdir "/etc/pve/nodes/$nodename";
 143 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 144 mkdir $confdir;
 145
 146 my $var_run_tmpdir = "/var/run/qemu-server";
 147 mkdir $var_run_tmpdir;
 148
 149 my $lock_dir = "/var/lock/qemu-server";
 150 mkdir $lock_dir;
 151
 152 my $pcisysfs = "/sys/bus/pci";
 153
 154 my $confdesc = {
 155     iothread => {
 156         optional => 1,
 157         type => 'boolean',
 158         description => "Enable iothread dataplane.",
 159         default => 0,
 160     },
 161     onboot => {
 162         optional => 1,
 163         type => 'boolean',
 164         description => "Specifies whether a VM will be started during system bootup.",
 165         default => 0,
 166     },
 167     autostart => {
 168         optional => 1,
 169         type => 'boolean',
 170         description => "Automatic restart after crash (currently ignored).",
 171         default => 0,
 172     },
 173     hotplug => {
 174         optional => 1,
 175         type => 'boolean',
 176         description => "Allow hotplug for disk and network device",
 177         default => 0,
 178     },
 179     reboot => {
 180         optional => 1,
 181         type => 'boolean',
 182         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 183         default => 1,
 184     },
 185     lock => {
 186         optional => 1,
 187         type => 'string',
 188         description => "Lock/unlock the VM.",
 189         enum => [qw(migrate backup snapshot rollback)],
 190     },
 191     cpulimit => {
 192         optional => 1,
 193         type => 'integer',
 194         description => "Limit of CPU usage in per cent. Note if the computer has 2 CPUs, it has total of 200% CPU time. Value '0' indicates no CPU limit.\n\nNOTE: This option is currently ignored.",
 195         minimum => 0,
 196         default => 0,
 197     },
 198     cpuunits => {
 199         optional => 1,
 200         type => 'integer',
 201         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 202         minimum => 0,
 203         maximum => 500000,
 204         default => 1000,
 205     },
 206     memory => {
 207         optional => 1,
 208         type => 'integer',
 209         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 210         minimum => 16,
 211         default => 512,
 212     },
 213     balloon => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 217         minimum => 0,
 218     },
 219     shares => {
 220         optional => 1,
 221         type => 'integer',
 222         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 223         minimum => 0,
 224         maximum => 50000,
 225         default => 1000,
 226     },
 227     keyboard => {
 228         optional => 1,
 229         type => 'string',
 230         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 231         enum => PVE::Tools::kvmkeymaplist(),
 232         default => 'en-us',
 233     },
 234     name => {
 235         optional => 1,
 236         type => 'string', format => 'dns-name',
 237         description => "Set a name for the VM. Only used on the configuration web interface.",
 238     },
 239     scsihw => {
 240         optional => 1,
 241         type => 'string',
 242         description => "scsi controller model",
 243         enum => [qw(lsi lsi53c810 virtio-scsi-pci megasas pvscsi)],
 244         default => 'lsi',
 245     },
 246     description => {
 247         optional => 1,
 248         type => 'string',
 249         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 250     },
 251     ostype => {
 252         optional => 1,
 253         type => 'string',
 254         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 255         description => <<EODESC,
 256 Used to enable special optimization/features for specific
 257 operating systems:
 258
 259 other  => unspecified OS
 260 wxp    => Microsoft Windows XP
 261 w2k    => Microsoft Windows 2000
 262 w2k3   => Microsoft Windows 2003
 263 w2k8   => Microsoft Windows 2008
 264 wvista => Microsoft Windows Vista
 265 win7   => Microsoft Windows 7
 266 win8   => Microsoft Windows 8/2012
 267 l24    => Linux 2.4 Kernel
 268 l26    => Linux 2.6/3.X Kernel
 269 solaris => solaris/opensolaris/openindiania kernel
 270
 271 other|l24|l26|solaris                       ... no special behaviour
 272 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 273 EODESC
 274     },
 275     boot => {
 276         optional => 1,
 277         type => 'string',
 278         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 279         pattern => '[acdn]{1,4}',
 280         default => 'cdn',
 281     },
 282     bootdisk => {
 283         optional => 1,
 284         type => 'string', format => 'pve-qm-bootdisk',
 285         description => "Enable booting from specified disk.",
 286         pattern => '(ide|sata|scsi|virtio)\d+',
 287     },
 288     smp => {
 289         optional => 1,
 290         type => 'integer',
 291         description => "The number of CPUs. Please use option -sockets instead.",
 292         minimum => 1,
 293         default => 1,
 294     },
 295     sockets => {
 296         optional => 1,
 297         type => 'integer',
 298         description => "The number of CPU sockets.",
 299         minimum => 1,
 300         default => 1,
 301     },
 302     cores => {
 303         optional => 1,
 304         type => 'integer',
 305         description => "The number of cores per socket.",
 306         minimum => 1,
 307         default => 1,
 308     },
 309     maxcpus => {
 310         optional => 1,
 311         type => 'integer',
 312         description => "Maximum cpus for hotplug.",
 313         minimum => 1,
 314         default => 1,
 315     },
 316     acpi => {
 317         optional => 1,
 318         type => 'boolean',
 319         description => "Enable/disable ACPI.",
 320         default => 1,
 321     },
 322     agent => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Enable/disable Qemu GuestAgent.",
 326         default => 0,
 327     },
 328     kvm => {
 329         optional => 1,
 330         type => 'boolean',
 331         description => "Enable/disable KVM hardware virtualization.",
 332         default => 1,
 333     },
 334     tdf => {
 335         optional => 1,
 336         type => 'boolean',
 337         description => "Enable/disable time drift fix.",
 338         default => 0,
 339     },
 340     localtime => {
 341         optional => 1,
 342         type => 'boolean',
 343         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 344     },
 345     freeze => {
 346         optional => 1,
 347         type => 'boolean',
 348         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 349     },
 350     vga => {
 351         optional => 1,
 352         type => 'string',
 353         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 354         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 355     },
 356     watchdog => {
 357         optional => 1,
 358         type => 'string', format => 'pve-qm-watchdog',
 359         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 360         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 361     },
 362     startdate => {
 363         optional => 1,
 364         type => 'string',
 365         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 366         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 367         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 368         default => 'now',
 369     },
 370     startup => {
 371         optional => 1,
 372         type => 'string', format => 'pve-qm-startup',
 373         typetext => '[[order=]\d+] [,up=\d+] [,down=\d+] ',
 374         description => "Startup and shutdown behavior. Order is a non-negative number defining the general startup order. Shutdown in done with reverse ordering. Additionally you can set the 'up' or 'down' delay in seconds, which specifies a delay to wait before the next VM is started or stopped.",
 375     },
 376     template => {
 377         optional => 1,
 378         type => 'boolean',
 379         description => "Enable/disable Template.",
 380         default => 0,
 381     },
 382     args => {
 383         optional => 1,
 384         type => 'string',
 385         description => <<EODESCR,
 386 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 387
 388 args: -no-reboot -no-hpet
 389 EODESCR
 390     },
 391     tablet => {
 392         optional => 1,
 393         type => 'boolean',
 394         default => 1,
 395         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 396     },
 397     migrate_speed => {
 398         optional => 1,
 399         type => 'integer',
 400         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 401         minimum => 0,
 402         default => 0,
 403     },
 404     migrate_downtime => {
 405         optional => 1,
 406         type => 'number',
 407         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 408         minimum => 0,
 409         default => 0.1,
 410     },
 411     cdrom => {
 412         optional => 1,
 413         type => 'string', format => 'pve-qm-drive',
 414         typetext => 'volume',
 415         description => "This is an alias for option -ide2",
 416     },
 417     cpu => {
 418         optional => 1,
 419         description => "Emulated CPU type.",
 420         type => 'string',
 421         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 422         default => 'kvm64',
 423     },
 424     parent => get_standard_option('pve-snapshot-name', {
 425         optional => 1,
 426         description => "Parent snapshot name. This is used internally, and should not be modified.",
 427     }),
 428     snaptime => {
 429         optional => 1,
 430         description => "Timestamp for snapshots.",
 431         type => 'integer',
 432         minimum => 0,
 433     },
 434     vmstate => {
 435         optional => 1,
 436         type => 'string', format => 'pve-volume-id',
 437         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 438     },
 439     machine => {
 440         description => "Specific the Qemu machine type.",
 441         type => 'string',
 442         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 443         maxLength => 40,
 444         optional => 1,
 445     },
 446     smbios1 => {
 447         description => "Specify SMBIOS type 1 fields.",
 448         type => 'string', format => 'pve-qm-smbios1',
 449         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 450         maxLength => 256,
 451         optional => 1,
 452     },
 453 };
 454
 455 # what about other qemu settings ?
 456 #cpu => 'string',
 457 #machine => 'string',
 458 #fda => 'file',
 459 #fdb => 'file',
 460 #mtdblock => 'file',
 461 #sd => 'file',
 462 #pflash => 'file',
 463 #snapshot => 'bool',
 464 #bootp => 'file',
 465 ##tftp => 'dir',
 466 ##smb => 'dir',
 467 #kernel => 'file',
 468 #append => 'string',
 469 #initrd => 'file',
 470 ##soundhw => 'string',
 471
 472 while (my ($k, $v) = each %$confdesc) {
 473     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 474 }
 475
 476 my $MAX_IDE_DISKS = 4;
 477 my $MAX_SCSI_DISKS = 14;
 478 my $MAX_VIRTIO_DISKS = 16;
 479 my $MAX_SATA_DISKS = 6;
 480 my $MAX_USB_DEVICES = 5;
 481 my $MAX_NETS = 32;
 482 my $MAX_UNUSED_DISKS = 8;
 483 my $MAX_HOSTPCI_DEVICES = 4;
 484 my $MAX_SERIAL_PORTS = 4;
 485 my $MAX_PARALLEL_PORTS = 3;
 486
 487 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 488                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3'];
 489 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 490
 491 my $netdesc = {
 492     optional => 1,
 493     type => 'string', format => 'pve-qm-net',
 494     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>][,tag=<vlanid>][,firewall=0|1]",
 495     description => <<EODESCR,
 496 Specify network devices.
 497
 498 MODEL is one of: $nic_model_list_txt
 499
 500 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 501 automatically generated if not specified.
 502
 503 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 504
 505 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 506
 507 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 508
 509 10.0.2.2   Gateway
 510 10.0.2.3   DNS Server
 511 10.0.2.4   SMB Server
 512
 513 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 514
 515 EODESCR
 516 };
 517 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 518
 519 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 520     $confdesc->{"net$i"} = $netdesc;
 521 }
 522
 523 my $drivename_hash;
 524
 525 my $idedesc = {
 526     optional => 1,
 527     type => 'string', format => 'pve-qm-drive',
 528     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 529     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 530 };
 531 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 532
 533 my $scsidesc = {
 534     optional => 1,
 535     type => 'string', format => 'pve-qm-drive',
 536     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 537     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 538 };
 539 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 540
 541 my $satadesc = {
 542     optional => 1,
 543     type => 'string', format => 'pve-qm-drive',
 544     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 545     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 546 };
 547 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 548
 549 my $virtiodesc = {
 550     optional => 1,
 551     type => 'string', format => 'pve-qm-drive',
 552     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 553     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 554 };
 555 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 556
 557 my $usbdesc = {
 558     optional => 1,
 559     type => 'string', format => 'pve-qm-usb-device',
 560     typetext => 'host=HOSTUSBDEVICE|spice',
 561     description => <<EODESCR,
 562 Configure an USB device (n is 0 to 4). This can be used to
 563 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 564
 565 'bus-port(.port)*' (decimal numbers) or
 566 'vendor_id:product_id' (hexadeciaml numbers)
 567
 568 You can use the 'lsusb -t' command to list existing usb devices.
 569
 570 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 571
 572 The value 'spice' can be used to add a usb redirection devices for spice.
 573
 574 EODESCR
 575 };
 576 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 577
 578 my $hostpcidesc = {
 579         optional => 1,
 580         type => 'string', format => 'pve-qm-hostpci',
 581         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 582         description => <<EODESCR,
 583 Map host pci devices. HOSTPCIDEVICE syntax is:
 584
 585 'bus:dev.func' (hexadecimal numbers)
 586
 587 You can us the 'lspci' command to list existing pci devices.
 588
 589 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 590
 591 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 592
 593 Experimental: user reported problems with this option.
 594 EODESCR
 595 };
 596 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 597
 598 my $serialdesc = {
 599         optional => 1,
 600         type => 'string',
 601         pattern => '(/dev/ttyS\d+|socket)',
 602         description =>  <<EODESCR,
 603 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device, or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 604
 605 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 606
 607 Experimental: user reported problems with this option.
 608 EODESCR
 609 };
 610
 611 my $paralleldesc= {
 612         optional => 1,
 613         type => 'string',
 614         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 615         description =>  <<EODESCR,
 616 Map host parallel devices (n is 0 to 2).
 617
 618 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 619
 620 Experimental: user reported problems with this option.
 621 EODESCR
 622 };
 623
 624 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 625     $confdesc->{"parallel$i"} = $paralleldesc;
 626 }
 627
 628 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 629     $confdesc->{"serial$i"} = $serialdesc;
 630 }
 631
 632 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 633     $confdesc->{"hostpci$i"} = $hostpcidesc;
 634 }
 635
 636 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 637     $drivename_hash->{"ide$i"} = 1;
 638     $confdesc->{"ide$i"} = $idedesc;
 639 }
 640
 641 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 642     $drivename_hash->{"sata$i"} = 1;
 643     $confdesc->{"sata$i"} = $satadesc;
 644 }
 645
 646 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 647     $drivename_hash->{"scsi$i"} = 1;
 648     $confdesc->{"scsi$i"} = $scsidesc ;
 649 }
 650
 651 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 652     $drivename_hash->{"virtio$i"} = 1;
 653     $confdesc->{"virtio$i"} = $virtiodesc;
 654 }
 655
 656 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 657     $confdesc->{"usb$i"} = $usbdesc;
 658 }
 659
 660 my $unuseddesc = {
 661     optional => 1,
 662     type => 'string', format => 'pve-volume-id',
 663     description => "Reference to unused volumes.",
 664 };
 665
 666 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 667     $confdesc->{"unused$i"} = $unuseddesc;
 668 }
 669
 670 my $kvm_api_version = 0;
 671
 672 sub kvm_version {
 673
 674     return $kvm_api_version if $kvm_api_version;
 675
 676     my $fh = IO::File->new("</dev/kvm") ||
 677         return 0;
 678
 679     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 680         $kvm_api_version = $v;
 681     }
 682
 683     $fh->close();
 684
 685     return  $kvm_api_version;
 686 }
 687
 688 my $kvm_user_version;
 689
 690 sub kvm_user_version {
 691
 692     return $kvm_user_version if $kvm_user_version;
 693
 694     $kvm_user_version = 'unknown';
 695
 696     my $tmp = `kvm -help 2>/dev/null`;
 697
 698     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 699         $kvm_user_version = $2;
 700     }
 701
 702     return $kvm_user_version;
 703
 704 }
 705
 706 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 707
 708 sub disknames {
 709     # order is important - used to autoselect boot disk
 710     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 711             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 712             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 713             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 714 }
 715
 716 sub valid_drivename {
 717     my $dev = shift;
 718
 719     return defined($drivename_hash->{$dev});
 720 }
 721
 722 sub option_exists {
 723     my $key = shift;
 724     return defined($confdesc->{$key});
 725 }
 726
 727 sub nic_models {
 728     return $nic_model_list;
 729 }
 730
 731 sub os_list_description {
 732
 733     return {
 734         other => 'Other',
 735         wxp => 'Windows XP',
 736         w2k => 'Windows 2000',
 737         w2k3 =>, 'Windows 2003',
 738         w2k8 => 'Windows 2008',
 739         wvista => 'Windows Vista',
 740         win7 => 'Windows 7',
 741         win8 => 'Windows 8/2012',
 742         l24 => 'Linux 2.4',
 743         l26 => 'Linux 2.6',
 744     };
 745 }
 746
 747 my $cdrom_path;
 748
 749 sub get_cdrom_path {
 750
 751     return  $cdrom_path if $cdrom_path;
 752
 753     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 754     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 755     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 756 }
 757
 758 sub get_iso_path {
 759     my ($storecfg, $vmid, $cdrom) = @_;
 760
 761     if ($cdrom eq 'cdrom') {
 762         return get_cdrom_path();
 763     } elsif ($cdrom eq 'none') {
 764         return '';
 765     } elsif ($cdrom =~ m|^/|) {
 766         return $cdrom;
 767     } else {
 768         return PVE::Storage::path($storecfg, $cdrom);
 769     }
 770 }
 771
 772 # try to convert old style file names to volume IDs
 773 sub filename_to_volume_id {
 774     my ($vmid, $file, $media) = @_;
 775
 776     if (!($file eq 'none' || $file eq 'cdrom' ||
 777           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 778
 779         return undef if $file =~ m|/|;
 780
 781         if ($media && $media eq 'cdrom') {
 782             $file = "local:iso/$file";
 783         } else {
 784             $file = "local:$vmid/$file";
 785         }
 786     }
 787
 788     return $file;
 789 }
 790
 791 sub verify_media_type {
 792     my ($opt, $vtype, $media) = @_;
 793
 794     return if !$media;
 795
 796     my $etype;
 797     if ($media eq 'disk') {
 798         $etype = 'images';
 799     } elsif ($media eq 'cdrom') {
 800         $etype = 'iso';
 801     } else {
 802         die "internal error";
 803     }
 804
 805     return if ($vtype eq $etype);
 806
 807     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 808 }
 809
 810 sub cleanup_drive_path {
 811     my ($opt, $storecfg, $drive) = @_;
 812
 813     # try to convert filesystem paths to volume IDs
 814
 815     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 816         ($drive->{file} !~ m|^/dev/.+|) &&
 817         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 818         ($drive->{file} !~ m/^\d+$/)) {
 819         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 820         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 821         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 822         verify_media_type($opt, $vtype, $drive->{media});
 823         $drive->{file} = $volid;
 824     }
 825
 826     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 827 }
 828
 829 sub create_conf_nolock {
 830     my ($vmid, $settings) = @_;
 831
 832     my $filename = config_file($vmid);
 833
 834     die "configuration file '$filename' already exists\n" if -f $filename;
 835
 836     my $defaults = load_defaults();
 837
 838     $settings->{name} = "vm$vmid" if !$settings->{name};
 839     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 840
 841     my $data = '';
 842     foreach my $opt (keys %$settings) {
 843         next if !$confdesc->{$opt};
 844
 845         my $value = $settings->{$opt};
 846         next if !$value;
 847
 848         $data .= "$opt: $value\n";
 849     }
 850
 851     PVE::Tools::file_set_contents($filename, $data);
 852 }
 853
 854 my $parse_size = sub {
 855     my ($value) = @_;
 856
 857     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 858     my ($size, $unit) = ($1, $3);
 859     if ($unit) {
 860         if ($unit eq 'K') {
 861             $size = $size * 1024;
 862         } elsif ($unit eq 'M') {
 863             $size = $size * 1024 * 1024;
 864         } elsif ($unit eq 'G') {
 865             $size = $size * 1024 * 1024 * 1024;
 866         }
 867     }
 868     return int($size);
 869 };
 870
 871 my $format_size = sub {
 872     my ($size) = @_;
 873
 874     $size = int($size);
 875
 876     my $kb = int($size/1024);
 877     return $size if $kb*1024 != $size;
 878
 879     my $mb = int($kb/1024);
 880     return "${kb}K" if $mb*1024 != $kb;
 881
 882     my $gb = int($mb/1024);
 883     return "${mb}M" if $gb*1024 != $mb;
 884
 885     return "${gb}G";
 886 };
 887
 888 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 889 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 890 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 891 #        [,aio=native|threads][,discard=ignore|on]
 892
 893 sub parse_drive {
 894     my ($key, $data) = @_;
 895
 896     my $res = {};
 897
 898     # $key may be undefined - used to verify JSON parameters
 899     if (!defined($key)) {
 900         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 901         $res->{index} = 0;
 902     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 903         $res->{interface} = $1;
 904         $res->{index} = $2;
 905     } else {
 906         return undef;
 907     }
 908
 909     foreach my $p (split (/,/, $data)) {
 910         next if $p =~ m/^\s*$/;
 911
 912         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard)=(.+)$/) {
 913             my ($k, $v) = ($1, $2);
 914
 915             $k = 'file' if $k eq 'volume';
 916
 917             return undef if defined $res->{$k};
 918
 919             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 920                 return undef if !$v || $v !~ m/^\d+/;
 921                 $k = "m$k";
 922                 $v = sprintf("%.3f", $v / (1024*1024));
 923             }
 924             $res->{$k} = $v;
 925         } else {
 926             if (!$res->{file} && $p !~ m/=/) {
 927                 $res->{file} = $p;
 928             } else {
 929                 return undef;
 930             }
 931         }
 932     }
 933
 934     return undef if !$res->{file};
 935
 936     if($res->{file} =~ m/\.(raw|cow|qcow|qcow2|vmdk|cloop)$/){
 937         $res->{format} = $1;
 938     }
 939
 940     return undef if $res->{cache} &&
 941         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 942     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 943     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 944     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 945     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 946     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 947     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 948     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qcow2|vmdk|cloop)$/;
 949     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 950     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 951     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 952     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 953     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 954
 955     return undef if $res->{mbps_rd} && $res->{mbps};
 956     return undef if $res->{mbps_wr} && $res->{mbps};
 957
 958     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 959     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 960     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 961     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 962     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 963     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 964
 965     return undef if $res->{iops_rd} && $res->{iops};
 966     return undef if $res->{iops_wr} && $res->{iops};
 967
 968
 969     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 970     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 971     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 972     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 973     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 974     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 975
 976
 977     if ($res->{size}) {
 978         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 979     }
 980
 981     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 982         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 983         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 984         return undef if $res->{interface} eq 'virtio';
 985     }
 986
 987     # rerror does not work with scsi drives
 988     if ($res->{rerror}) {
 989         return undef if $res->{interface} eq 'scsi';
 990     }
 991
 992     return $res;
 993 }
 994
 995 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
 996
 997 sub print_drive {
 998     my ($vmid, $drive) = @_;
 999
1000     my $opts = '';
1001     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup') {
1002         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1003     }
1004
1005     if ($drive->{size}) {
1006         $opts .= ",size=" . &$format_size($drive->{size});
1007     }
1008
1009     return "$drive->{file}$opts";
1010 }
1011
1012 sub scsi_inquiry {
1013     my($fh, $noerr) = @_;
1014
1015     my $SG_IO = 0x2285;
1016     my $SG_GET_VERSION_NUM = 0x2282;
1017
1018     my $versionbuf = "\x00" x 8;
1019     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1020     if (!$ret) {
1021         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1022         return undef;
1023     }
1024     my $version = unpack("I", $versionbuf);
1025     if ($version < 30000) {
1026         die "scsi generic interface too old\n"  if !$noerr;
1027         return undef;
1028     }
1029
1030     my $buf = "\x00" x 36;
1031     my $sensebuf = "\x00" x 8;
1032     my $cmd = pack("C x3 C x1", 0x12, 36);
1033
1034     # see /usr/include/scsi/sg.h
1035     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1036
1037     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1038                       length($sensebuf), 0, length($buf), $buf,
1039                       $cmd, $sensebuf, 6000);
1040
1041     $ret = ioctl($fh, $SG_IO, $packet);
1042     if (!$ret) {
1043         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1044         return undef;
1045     }
1046
1047     my @res = unpack($sg_io_hdr_t, $packet);
1048     if ($res[17] || $res[18]) {
1049         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1050         return undef;
1051     }
1052
1053     my $res = {};
1054     (my $byte0, my $byte1, $res->{vendor},
1055      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1056
1057     $res->{removable} = $byte1 & 128 ? 1 : 0;
1058     $res->{type} = $byte0 & 31;
1059
1060     return $res;
1061 }
1062
1063 sub path_is_scsi {
1064     my ($path) = @_;
1065
1066     my $fh = IO::File->new("+<$path") || return undef;
1067     my $res = scsi_inquiry($fh, 1);
1068     close($fh);
1069
1070     return $res;
1071 }
1072
1073 sub machine_type_is_q35 {
1074     my ($conf) = @_;
1075
1076     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1077 }
1078
1079 sub print_tabletdevice_full {
1080     my ($conf) = @_;
1081
1082     my $q35 = machine_type_is_q35($conf);
1083
1084     # we use uhci for old VMs because tablet driver was buggy in older qemu
1085     my $usbbus = $q35 ? "ehci" : "uhci";
1086
1087     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1088 }
1089
1090 sub print_drivedevice_full {
1091     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1092
1093     my $device = '';
1094     my $maxdev = 0;
1095
1096     if ($drive->{interface} eq 'virtio') {
1097         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1098         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1099         $device .= ",iothread=iothread0" if $conf->{iothread};
1100     } elsif ($drive->{interface} eq 'scsi') {
1101         $maxdev = ($conf->{scsihw} && ($conf->{scsihw} !~ m/^lsi/)) ? 256 : 7;
1102         my $controller = int($drive->{index} / $maxdev);
1103         my $unit = $drive->{index} % $maxdev;
1104         my $devicetype = 'hd';
1105         my $path = '';
1106         if (drive_is_cdrom($drive)) {
1107               $devicetype = 'cd';
1108         } else {
1109               if ($drive->{file} =~ m|^/|) {
1110                   $path = $drive->{file};
1111               } else {
1112                   $path = PVE::Storage::path($storecfg, $drive->{file});
1113               }
1114
1115               if($path =~ m/^iscsi\:\/\//){
1116                   $devicetype = 'generic';
1117               } else {
1118                   if (my $info = path_is_scsi($path)) {
1119                       if ($info->{type} == 0) {
1120                           $devicetype = 'block';
1121                       } elsif ($info->{type} == 1) { # tape
1122                           $devicetype = 'generic';
1123                       }
1124                   }
1125               }
1126          }
1127
1128         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1129             $device = "scsi-$devicetype,bus=scsihw$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1130         } else {
1131             $device = "scsi-$devicetype,bus=scsihw$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1132         }
1133
1134     } elsif ($drive->{interface} eq 'ide'){
1135         $maxdev = 2;
1136         my $controller = int($drive->{index} / $maxdev);
1137         my $unit = $drive->{index} % $maxdev;
1138         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1139
1140         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1141     } elsif ($drive->{interface} eq 'sata'){
1142         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1143         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1144         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1145     } elsif ($drive->{interface} eq 'usb') {
1146         die "implement me";
1147         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1148     } else {
1149         die "unsupported interface type";
1150     }
1151
1152     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1153
1154     return $device;
1155 }
1156
1157 sub get_initiator_name {
1158     my $initiator;
1159
1160     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1161     while (defined(my $line = <$fh>)) {
1162         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1163         $initiator = $1;
1164         last;
1165     }
1166     $fh->close();
1167
1168     return $initiator;
1169 }
1170
1171 sub print_drive_full {
1172     my ($storecfg, $vmid, $drive) = @_;
1173
1174     my $opts = '';
1175     foreach my $o (@qemu_drive_options) {
1176         next if $o eq 'bootindex';
1177         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1178     }
1179
1180     foreach my $o (qw(bps bps_rd bps_wr)) {
1181         my $v = $drive->{"m$o"};
1182         $opts .= ",$o=" . int($v*1024*1024) if $v;
1183     }
1184
1185     # use linux-aio by default (qemu default is threads)
1186     $opts .= ",aio=native" if !$drive->{aio};
1187
1188     my $path;
1189     my $volid = $drive->{file};
1190     if (drive_is_cdrom($drive)) {
1191         $path = get_iso_path($storecfg, $vmid, $volid);
1192     } else {
1193         if ($volid =~ m|^/|) {
1194             $path = $volid;
1195         } else {
1196             $path = PVE::Storage::path($storecfg, $volid);
1197         }
1198     }
1199
1200     $opts .= ",cache=none" if !$drive->{cache} && !drive_is_cdrom($drive);
1201
1202     my $pathinfo = $path ? "file=$path," : '';
1203
1204     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1205 }
1206
1207 sub print_netdevice_full {
1208     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1209
1210     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1211
1212     my $device = $net->{model};
1213     if ($net->{model} eq 'virtio') {
1214          $device = 'virtio-net-pci';
1215      };
1216
1217     # qemu > 0.15 always try to boot from network - we disable that by
1218     # not loading the pxe rom file
1219     my $extra = ($bootorder !~ m/n/) ? "romfile=," : '';
1220     my $pciaddr = print_pci_addr("$netid", $bridges);
1221     my $tmpstr = "$device,${extra}mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1222     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1223         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1224         my $vectors = $net->{queues} * 2 + 2;
1225         $tmpstr .= ",vectors=$vectors,mq=on";
1226     }
1227     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1228     return $tmpstr;
1229 }
1230
1231 sub print_netdev_full {
1232     my ($vmid, $conf, $net, $netid) = @_;
1233
1234     my $i = '';
1235     if ($netid =~ m/^net(\d+)$/) {
1236         $i = int($1);
1237     }
1238
1239     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1240
1241     my $ifname = "tap${vmid}i$i";
1242
1243     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1244     die "interface name '$ifname' is too long (max 15 character)\n"
1245         if length($ifname) >= 16;
1246
1247     my $vhostparam = '';
1248     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1249
1250     my $vmname = $conf->{name} || "vm$vmid";
1251
1252     my $netdev = "";
1253
1254     if ($net->{bridge}) {
1255         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1256     } else {
1257         $netdev = "type=user,id=$netid,hostname=$vmname";
1258     }
1259
1260     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1261
1262     return $netdev;
1263 }
1264
1265 sub drive_is_cdrom {
1266     my ($drive) = @_;
1267
1268     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1269
1270 }
1271
1272 sub parse_hostpci {
1273     my ($value) = @_;
1274
1275     return undef if !$value;
1276
1277
1278     my @list = split(/,/, $value);
1279     my $found;
1280
1281     my $res = {};
1282     foreach my $kv (@list) {
1283
1284         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1285             $found = 1;
1286             if(defined($4)){
1287                 push @{$res->{pciid}}, { id => $2 , function => $4};
1288
1289             }else{
1290                 my $pcidevices = lspci($2);
1291                 $res->{pciid} = $pcidevices->{$2};
1292             }
1293         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1294             $res->{driver} = $1;
1295         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1296             $res->{rombar} = $1;
1297         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1298             $res->{'x-vga'} = $1;
1299         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1300             $res->{pcie} = 1 if $1 == 1;
1301         } else {
1302             warn "unknown hostpci setting '$kv'\n";
1303         }
1304     }
1305
1306     return undef if !$found;
1307
1308     return $res;
1309 }
1310
1311 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1312 sub parse_net {
1313     my ($data) = @_;
1314
1315     my $res = {};
1316
1317     foreach my $kvp (split(/,/, $data)) {
1318
1319         if ($kvp =~ m/^(ne2k_pci|e1000|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1320             my $model = lc($1);
1321             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1322             $res->{model} = $model;
1323             $res->{macaddr} = $mac;
1324         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1325             $res->{bridge} = $1;
1326         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1327             $res->{queues} = $1;
1328         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1329             $res->{rate} = $1;
1330         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1331             $res->{tag} = $1;
1332         } elsif ($kvp =~ m/^firewall=(\d+)$/) {
1333             $res->{firewall} = $1;
1334         } else {
1335             return undef;
1336         }
1337
1338     }
1339
1340     return undef if !$res->{model};
1341
1342     return $res;
1343 }
1344
1345 sub print_net {
1346     my $net = shift;
1347
1348     my $res = "$net->{model}";
1349     $res .= "=$net->{macaddr}" if $net->{macaddr};
1350     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1351     $res .= ",rate=$net->{rate}" if $net->{rate};
1352     $res .= ",tag=$net->{tag}" if $net->{tag};
1353     $res .= ",firewall=$net->{firewall}" if $net->{firewall};
1354
1355     return $res;
1356 }
1357
1358 sub add_random_macs {
1359     my ($settings) = @_;
1360
1361     foreach my $opt (keys %$settings) {
1362         next if $opt !~ m/^net(\d+)$/;
1363         my $net = parse_net($settings->{$opt});
1364         next if !$net;
1365         $settings->{$opt} = print_net($net);
1366     }
1367 }
1368
1369 sub add_unused_volume {
1370     my ($config, $volid) = @_;
1371
1372     my $key;
1373     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1374         my $test = "unused$ind";
1375         if (my $vid = $config->{$test}) {
1376             return if $vid eq $volid; # do not add duplicates
1377         } else {
1378             $key = $test;
1379         }
1380     }
1381
1382     die "To many unused volume - please delete them first.\n" if !$key;
1383
1384     $config->{$key} = $volid;
1385
1386     return $key;
1387 }
1388
1389 my $valid_smbios1_options = {
1390     manufacturer => '\S+',
1391     product => '\S+',
1392     version => '\S+',
1393     serial => '\S+',
1394     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1395     sku => '\S+',
1396     family => '\S+',
1397 };
1398
1399 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1400 sub parse_smbios1 {
1401     my ($data) = @_;
1402
1403     my $res = {};
1404
1405     foreach my $kvp (split(/,/, $data)) {
1406         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1407         my ($k, $v) = split(/=/, $kvp);
1408         return undef if !defined($k) || !defined($v);
1409         return undef if !$valid_smbios1_options->{$k};
1410         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1411         $res->{$k} = $v;
1412     }
1413
1414     return $res;
1415 }
1416
1417 sub print_smbios1 {
1418     my ($smbios1) = @_;
1419
1420     my $data = '';
1421     foreach my $k (keys %$smbios1) {
1422         next if !defined($smbios1->{$k});
1423         next if !$valid_smbios1_options->{$k};
1424         $data .= ',' if $data;
1425         $data .= "$k=$smbios1->{$k}";
1426     }
1427     return $data;
1428 }
1429
1430 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1431 sub verify_smbios1 {
1432     my ($value, $noerr) = @_;
1433
1434     return $value if parse_smbios1($value);
1435
1436     return undef if $noerr;
1437
1438     die "unable to parse smbios (type 1) options\n";
1439 }
1440
1441 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1442 sub verify_bootdisk {
1443     my ($value, $noerr) = @_;
1444
1445     return $value if valid_drivename($value);
1446
1447     return undef if $noerr;
1448
1449     die "invalid boot disk '$value'\n";
1450 }
1451
1452 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1453 sub verify_net {
1454     my ($value, $noerr) = @_;
1455
1456     return $value if parse_net($value);
1457
1458     return undef if $noerr;
1459
1460     die "unable to parse network options\n";
1461 }
1462
1463 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1464 sub verify_drive {
1465     my ($value, $noerr) = @_;
1466
1467     return $value if parse_drive(undef, $value);
1468
1469     return undef if $noerr;
1470
1471     die "unable to parse drive options\n";
1472 }
1473
1474 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1475 sub verify_hostpci {
1476     my ($value, $noerr) = @_;
1477
1478     return $value if parse_hostpci($value);
1479
1480     return undef if $noerr;
1481
1482     die "unable to parse pci id\n";
1483 }
1484
1485 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1486 sub verify_watchdog {
1487     my ($value, $noerr) = @_;
1488
1489     return $value if parse_watchdog($value);
1490
1491     return undef if $noerr;
1492
1493     die "unable to parse watchdog options\n";
1494 }
1495
1496 sub parse_watchdog {
1497     my ($value) = @_;
1498
1499     return undef if !$value;
1500
1501     my $res = {};
1502
1503     foreach my $p (split(/,/, $value)) {
1504         next if $p =~ m/^\s*$/;
1505
1506         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1507             $res->{model} = $2;
1508         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1509             $res->{action} = $2;
1510         } else {
1511             return undef;
1512         }
1513     }
1514
1515     return $res;
1516 }
1517
1518 PVE::JSONSchema::register_format('pve-qm-startup', \&verify_startup);
1519 sub verify_startup {
1520     my ($value, $noerr) = @_;
1521
1522     return $value if parse_startup($value);
1523
1524     return undef if $noerr;
1525
1526     die "unable to parse startup options\n";
1527 }
1528
1529 sub parse_startup {
1530     my ($value) = @_;
1531
1532     return undef if !$value;
1533
1534     my $res = {};
1535
1536     foreach my $p (split(/,/, $value)) {
1537         next if $p =~ m/^\s*$/;
1538
1539         if ($p =~ m/^(order=)?(\d+)$/) {
1540             $res->{order} = $2;
1541         } elsif ($p =~ m/^up=(\d+)$/) {
1542             $res->{up} = $1;
1543         } elsif ($p =~ m/^down=(\d+)$/) {
1544             $res->{down} = $1;
1545         } else {
1546             return undef;
1547         }
1548     }
1549
1550     return $res;
1551 }
1552
1553 sub parse_usb_device {
1554     my ($value) = @_;
1555
1556     return undef if !$value;
1557
1558     my @dl = split(/,/, $value);
1559     my $found;
1560
1561     my $res = {};
1562     foreach my $v (@dl) {
1563         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1564             $found = 1;
1565             $res->{vendorid} = $2;
1566             $res->{productid} = $4;
1567         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1568             $found = 1;
1569             $res->{hostbus} = $1;
1570             $res->{hostport} = $2;
1571         } elsif ($v =~ m/^spice$/) {
1572             $found = 1;
1573             $res->{spice} = 1;
1574         } else {
1575             return undef;
1576         }
1577     }
1578     return undef if !$found;
1579
1580     return $res;
1581 }
1582
1583 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1584 sub verify_usb_device {
1585     my ($value, $noerr) = @_;
1586
1587     return $value if parse_usb_device($value);
1588
1589     return undef if $noerr;
1590
1591     die "unable to parse usb device\n";
1592 }
1593
1594 # add JSON properties for create and set function
1595 sub json_config_properties {
1596     my $prop = shift;
1597
1598     foreach my $opt (keys %$confdesc) {
1599         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1600         $prop->{$opt} = $confdesc->{$opt};
1601     }
1602
1603     return $prop;
1604 }
1605
1606 sub check_type {
1607     my ($key, $value) = @_;
1608
1609     die "unknown setting '$key'\n" if !$confdesc->{$key};
1610
1611     my $type = $confdesc->{$key}->{type};
1612
1613     if (!defined($value)) {
1614         die "got undefined value\n";
1615     }
1616
1617     if ($value =~ m/[\n\r]/) {
1618         die "property contains a line feed\n";
1619     }
1620
1621     if ($type eq 'boolean') {
1622         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1623         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1624         die "type check ('boolean') failed - got '$value'\n";
1625     } elsif ($type eq 'integer') {
1626         return int($1) if $value =~ m/^(\d+)$/;
1627         die "type check ('integer') failed - got '$value'\n";
1628     } elsif ($type eq 'number') {
1629         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1630         die "type check ('number') failed - got '$value'\n";
1631     } elsif ($type eq 'string') {
1632         if (my $fmt = $confdesc->{$key}->{format}) {
1633             if ($fmt eq 'pve-qm-drive') {
1634                 # special case - we need to pass $key to parse_drive()
1635                 my $drive = parse_drive($key, $value);
1636                 return $value if $drive;
1637                 die "unable to parse drive options\n";
1638             }
1639             PVE::JSONSchema::check_format($fmt, $value);
1640             return $value;
1641         }
1642         $value =~ s/^\"(.*)\"$/$1/;
1643         return $value;
1644     } else {
1645         die "internal error"
1646     }
1647 }
1648
1649 sub lock_config_full {
1650     my ($vmid, $timeout, $code, @param) = @_;
1651
1652     my $filename = config_file_lock($vmid);
1653
1654     my $res = lock_file($filename, $timeout, $code, @param);
1655
1656     die $@ if $@;
1657
1658     return $res;
1659 }
1660
1661 sub lock_config_mode {
1662     my ($vmid, $timeout, $shared, $code, @param) = @_;
1663
1664     my $filename = config_file_lock($vmid);
1665
1666     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1667
1668     die $@ if $@;
1669
1670     return $res;
1671 }
1672
1673 sub lock_config {
1674     my ($vmid, $code, @param) = @_;
1675
1676     return lock_config_full($vmid, 10, $code, @param);
1677 }
1678
1679 sub cfs_config_path {
1680     my ($vmid, $node) = @_;
1681
1682     $node = $nodename if !$node;
1683     return "nodes/$node/qemu-server/$vmid.conf";
1684 }
1685
1686 sub check_iommu_support{
1687     #fixme : need to check IOMMU support
1688     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1689
1690     my $iommu=1;
1691     return $iommu;
1692
1693 }
1694
1695 sub config_file {
1696     my ($vmid, $node) = @_;
1697
1698     my $cfspath = cfs_config_path($vmid, $node);
1699     return "/etc/pve/$cfspath";
1700 }
1701
1702 sub config_file_lock {
1703     my ($vmid) = @_;
1704
1705     return "$lock_dir/lock-$vmid.conf";
1706 }
1707
1708 sub touch_config {
1709     my ($vmid) = @_;
1710
1711     my $conf = config_file($vmid);
1712     utime undef, undef, $conf;
1713 }
1714
1715 sub destroy_vm {
1716     my ($storecfg, $vmid, $keep_empty_config) = @_;
1717
1718     my $conffile = config_file($vmid);
1719
1720     my $conf = load_config($vmid);
1721
1722     check_lock($conf);
1723
1724     # only remove disks owned by this VM
1725     foreach_drive($conf, sub {
1726         my ($ds, $drive) = @_;
1727
1728         return if drive_is_cdrom($drive);
1729
1730         my $volid = $drive->{file};
1731
1732         return if !$volid || $volid =~ m|^/|;
1733
1734         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1735         return if !$path || !$owner || ($owner != $vmid);
1736
1737         PVE::Storage::vdisk_free($storecfg, $volid);
1738     });
1739
1740     if ($keep_empty_config) {
1741         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1742     } else {
1743         unlink $conffile;
1744     }
1745
1746     # also remove unused disk
1747     eval {
1748         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1749
1750         eval {
1751             PVE::Storage::foreach_volid($dl, sub {
1752                 my ($volid, $sid, $volname, $d) = @_;
1753                 PVE::Storage::vdisk_free($storecfg, $volid);
1754             });
1755         };
1756         warn $@ if $@;
1757
1758     };
1759     warn $@ if $@;
1760 }
1761
1762 sub load_config {
1763     my ($vmid, $node) = @_;
1764
1765     my $cfspath = cfs_config_path($vmid, $node);
1766
1767     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1768
1769     die "no such VM ('$vmid')\n" if !defined($conf);
1770
1771     return $conf;
1772 }
1773
1774 sub parse_vm_config {
1775     my ($filename, $raw) = @_;
1776
1777     return undef if !defined($raw);
1778
1779     my $res = {
1780         digest => Digest::SHA::sha1_hex($raw),
1781         snapshots => {},
1782     };
1783
1784     $filename =~ m|/qemu-server/(\d+)\.conf$|
1785         || die "got strange filename '$filename'";
1786
1787     my $vmid = $1;
1788
1789     my $conf = $res;
1790     my $descr = '';
1791
1792     my @lines = split(/\n/, $raw);
1793     foreach my $line (@lines) {
1794         next if $line =~ m/^\s*$/;
1795
1796         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1797             my $snapname = $1;
1798             $conf->{description} = $descr if $descr;
1799             $descr = '';
1800             $conf = $res->{snapshots}->{$snapname} = {};
1801             next;
1802         }
1803
1804         if ($line =~ m/^\#(.*)\s*$/) {
1805             $descr .= PVE::Tools::decode_text($1) . "\n";
1806             next;
1807         }
1808
1809         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1810             $descr .= PVE::Tools::decode_text($2);
1811         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1812             $conf->{snapstate} = $1;
1813         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1814             my $key = $1;
1815             my $value = $2;
1816             $conf->{$key} = $value;
1817         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1818             my $key = $1;
1819             my $value = $2;
1820             eval { $value = check_type($key, $value); };
1821             if ($@) {
1822                 warn "vm $vmid - unable to parse value of '$key' - $@";
1823             } else {
1824                 my $fmt = $confdesc->{$key}->{format};
1825                 if ($fmt && $fmt eq 'pve-qm-drive') {
1826                     my $v = parse_drive($key, $value);
1827                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1828                         $v->{file} = $volid;
1829                         $value = print_drive($vmid, $v);
1830                     } else {
1831                         warn "vm $vmid - unable to parse value of '$key'\n";
1832                         next;
1833                     }
1834                 }
1835
1836                 if ($key eq 'cdrom') {
1837                     $conf->{ide2} = $value;
1838                 } else {
1839                     $conf->{$key} = $value;
1840                 }
1841             }
1842         }
1843     }
1844
1845     $conf->{description} = $descr if $descr;
1846
1847     delete $res->{snapstate}; # just to be sure
1848
1849     return $res;
1850 }
1851
1852 sub write_vm_config {
1853     my ($filename, $conf) = @_;
1854
1855     delete $conf->{snapstate}; # just to be sure
1856
1857     if ($conf->{cdrom}) {
1858         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
1859         $conf->{ide2} = $conf->{cdrom};
1860         delete $conf->{cdrom};
1861     }
1862
1863     # we do not use 'smp' any longer
1864     if ($conf->{sockets}) {
1865         delete $conf->{smp};
1866     } elsif ($conf->{smp}) {
1867         $conf->{sockets} = $conf->{smp};
1868         delete $conf->{cores};
1869         delete $conf->{smp};
1870     }
1871
1872     if ($conf->{maxcpus} && $conf->{sockets}) {
1873         delete $conf->{sockets};
1874     }
1875
1876     my $used_volids = {};
1877
1878     my $cleanup_config = sub {
1879         my ($cref, $snapname) = @_;
1880
1881         foreach my $key (keys %$cref) {
1882             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
1883                 $key eq 'snapstate';
1884             my $value = $cref->{$key};
1885             eval { $value = check_type($key, $value); };
1886             die "unable to parse value of '$key' - $@" if $@;
1887
1888             $cref->{$key} = $value;
1889
1890             if (!$snapname && valid_drivename($key)) {
1891                 my $drive = parse_drive($key, $value);
1892                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
1893             }
1894         }
1895     };
1896
1897     &$cleanup_config($conf);
1898     foreach my $snapname (keys %{$conf->{snapshots}}) {
1899         &$cleanup_config($conf->{snapshots}->{$snapname}, $snapname);
1900     }
1901
1902     # remove 'unusedX' settings if we re-add a volume
1903     foreach my $key (keys %$conf) {
1904         my $value = $conf->{$key};
1905         if ($key =~ m/^unused/ && $used_volids->{$value}) {
1906             delete $conf->{$key};
1907         }
1908     }
1909
1910     my $generate_raw_config = sub {
1911         my ($conf) = @_;
1912
1913         my $raw = '';
1914
1915         # add description as comment to top of file
1916         my $descr = $conf->{description} || '';
1917         foreach my $cl (split(/\n/, $descr)) {
1918             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
1919         }
1920
1921         foreach my $key (sort keys %$conf) {
1922             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots';
1923             $raw .= "$key: $conf->{$key}\n";
1924         }
1925         return $raw;
1926     };
1927
1928     my $raw = &$generate_raw_config($conf);
1929     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
1930         $raw .= "\n[$snapname]\n";
1931         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
1932     }
1933
1934     return $raw;
1935 }
1936
1937 sub update_config_nolock {
1938     my ($vmid, $conf, $skiplock) = @_;
1939
1940     check_lock($conf) if !$skiplock;
1941
1942     my $cfspath = cfs_config_path($vmid);
1943
1944     PVE::Cluster::cfs_write_file($cfspath, $conf);
1945 }
1946
1947 sub update_config {
1948     my ($vmid, $conf, $skiplock) = @_;
1949
1950     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
1951 }
1952
1953 sub load_defaults {
1954
1955     my $res = {};
1956
1957     # we use static defaults from our JSON schema configuration
1958     foreach my $key (keys %$confdesc) {
1959         if (defined(my $default = $confdesc->{$key}->{default})) {
1960             $res->{$key} = $default;
1961         }
1962     }
1963
1964     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
1965     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
1966
1967     return $res;
1968 }
1969
1970 sub config_list {
1971     my $vmlist = PVE::Cluster::get_vmlist();
1972     my $res = {};
1973     return $res if !$vmlist || !$vmlist->{ids};
1974     my $ids = $vmlist->{ids};
1975
1976     foreach my $vmid (keys %$ids) {
1977         my $d = $ids->{$vmid};
1978         next if !$d->{node} || $d->{node} ne $nodename;
1979         next if !$d->{type} || $d->{type} ne 'qemu';
1980         $res->{$vmid}->{exists} = 1;
1981     }
1982     return $res;
1983 }
1984
1985 # test if VM uses local resources (to prevent migration)
1986 sub check_local_resources {
1987     my ($conf, $noerr) = @_;
1988
1989     my $loc_res = 0;
1990
1991     $loc_res = 1 if $conf->{hostusb}; # old syntax
1992     $loc_res = 1 if $conf->{hostpci}; # old syntax
1993
1994     foreach my $k (keys %$conf) {
1995         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
1996         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
1997     }
1998
1999     die "VM uses local resources\n" if $loc_res && !$noerr;
2000
2001     return $loc_res;
2002 }
2003
2004 # check if used storages are available on all nodes (use by migrate)
2005 sub check_storage_availability {
2006     my ($storecfg, $conf, $node) = @_;
2007
2008     foreach_drive($conf, sub {
2009         my ($ds, $drive) = @_;
2010
2011         my $volid = $drive->{file};
2012         return if !$volid;
2013
2014         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2015         return if !$sid;
2016
2017         # check if storage is available on both nodes
2018         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2019         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2020    });
2021 }
2022
2023 # list nodes where all VM images are available (used by has_feature API)
2024 sub shared_nodes {
2025     my ($conf, $storecfg) = @_;
2026
2027     my $nodelist = PVE::Cluster::get_nodelist();
2028     my $nodehash = { map { $_ => 1 } @$nodelist };
2029     my $nodename = PVE::INotify::nodename();
2030
2031     foreach_drive($conf, sub {
2032         my ($ds, $drive) = @_;
2033
2034         my $volid = $drive->{file};
2035         return if !$volid;
2036
2037         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2038         if ($storeid) {
2039             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2040             if ($scfg->{disable}) {
2041                 $nodehash = {};
2042             } elsif (my $avail = $scfg->{nodes}) {
2043                 foreach my $node (keys %$nodehash) {
2044                     delete $nodehash->{$node} if !$avail->{$node};
2045                 }
2046             } elsif (!$scfg->{shared}) {
2047                 foreach my $node (keys %$nodehash) {
2048                     delete $nodehash->{$node} if $node ne $nodename
2049                 }
2050             }
2051         }
2052     });
2053
2054     return $nodehash
2055 }
2056
2057 sub check_lock {
2058     my ($conf) = @_;
2059
2060     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2061 }
2062
2063 sub check_cmdline {
2064     my ($pidfile, $pid) = @_;
2065
2066     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2067     if (defined($fh)) {
2068         my $line = <$fh>;
2069         $fh->close;
2070         return undef if !$line;
2071         my @param = split(/\0/, $line);
2072
2073         my $cmd = $param[0];
2074         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2075
2076         for (my $i = 0; $i < scalar (@param); $i++) {
2077             my $p = $param[$i];
2078             next if !$p;
2079             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2080                 my $p = $param[$i+1];
2081                 return 1 if $p && ($p eq $pidfile);
2082                 return undef;
2083             }
2084         }
2085     }
2086     return undef;
2087 }
2088
2089 sub check_running {
2090     my ($vmid, $nocheck, $node) = @_;
2091
2092     my $filename = config_file($vmid, $node);
2093
2094     die "unable to find configuration file for VM $vmid - no such machine\n"
2095         if !$nocheck && ! -f $filename;
2096
2097     my $pidfile = pidfile_name($vmid);
2098
2099     if (my $fd = IO::File->new("<$pidfile")) {
2100         my $st = stat($fd);
2101         my $line = <$fd>;
2102         close($fd);
2103
2104         my $mtime = $st->mtime;
2105         if ($mtime > time()) {
2106             warn "file '$filename' modified in future\n";
2107         }
2108
2109         if ($line =~ m/^(\d+)$/) {
2110             my $pid = $1;
2111             if (check_cmdline($pidfile, $pid)) {
2112                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2113                     return $pid;
2114                 }
2115             }
2116         }
2117     }
2118
2119     return undef;
2120 }
2121
2122 sub vzlist {
2123
2124     my $vzlist = config_list();
2125
2126     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2127
2128     while (defined(my $de = $fd->read)) {
2129         next if $de !~ m/^(\d+)\.pid$/;
2130         my $vmid = $1;
2131         next if !defined($vzlist->{$vmid});
2132         if (my $pid = check_running($vmid)) {
2133             $vzlist->{$vmid}->{pid} = $pid;
2134         }
2135     }
2136
2137     return $vzlist;
2138 }
2139
2140 sub disksize {
2141     my ($storecfg, $conf) = @_;
2142
2143     my $bootdisk = $conf->{bootdisk};
2144     return undef if !$bootdisk;
2145     return undef if !valid_drivename($bootdisk);
2146
2147     return undef if !$conf->{$bootdisk};
2148
2149     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2150     return undef if !defined($drive);
2151
2152     return undef if drive_is_cdrom($drive);
2153
2154     my $volid = $drive->{file};
2155     return undef if !$volid;
2156
2157     return $drive->{size};
2158 }
2159
2160 my $last_proc_pid_stat;
2161
2162 # get VM status information
2163 # This must be fast and should not block ($full == false)
2164 # We only query KVM using QMP if $full == true (this can be slow)
2165 sub vmstatus {
2166     my ($opt_vmid, $full) = @_;
2167
2168     my $res = {};
2169
2170     my $storecfg = PVE::Storage::config();
2171
2172     my $list = vzlist();
2173     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2174
2175     my $cpucount = $cpuinfo->{cpus} || 1;
2176
2177     foreach my $vmid (keys %$list) {
2178         next if $opt_vmid && ($vmid ne $opt_vmid);
2179
2180         my $cfspath = cfs_config_path($vmid);
2181         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2182
2183         my $d = {};
2184         $d->{pid} = $list->{$vmid}->{pid};
2185
2186         # fixme: better status?
2187         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2188
2189         my $size = disksize($storecfg, $conf);
2190         if (defined($size)) {
2191             $d->{disk} = 0; # no info available
2192             $d->{maxdisk} = $size;
2193         } else {
2194             $d->{disk} = 0;
2195             $d->{maxdisk} = 0;
2196         }
2197
2198         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2199         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2200
2201         $d->{name} = $conf->{name} || "VM $vmid";
2202         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2203
2204         if ($conf->{balloon}) {
2205             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2206             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2207         }
2208
2209         $d->{uptime} = 0;
2210         $d->{cpu} = 0;
2211         $d->{mem} = 0;
2212
2213         $d->{netout} = 0;
2214         $d->{netin} = 0;
2215
2216         $d->{diskread} = 0;
2217         $d->{diskwrite} = 0;
2218
2219         $d->{template} = is_template($conf);
2220
2221         $res->{$vmid} = $d;
2222     }
2223
2224     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2225     foreach my $dev (keys %$netdev) {
2226         next if $dev !~ m/^tap([1-9]\d*)i/;
2227         my $vmid = $1;
2228         my $d = $res->{$vmid};
2229         next if !$d;
2230
2231         $d->{netout} += $netdev->{$dev}->{receive};
2232         $d->{netin} += $netdev->{$dev}->{transmit};
2233     }
2234
2235     my $ctime = gettimeofday;
2236
2237     foreach my $vmid (keys %$list) {
2238
2239         my $d = $res->{$vmid};
2240         my $pid = $d->{pid};
2241         next if !$pid;
2242
2243         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2244         next if !$pstat; # not running
2245
2246         my $used = $pstat->{utime} + $pstat->{stime};
2247
2248         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2249
2250         if ($pstat->{vsize}) {
2251             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2252         }
2253
2254         my $old = $last_proc_pid_stat->{$pid};
2255         if (!$old) {
2256             $last_proc_pid_stat->{$pid} = {
2257                 time => $ctime,
2258                 used => $used,
2259                 cpu => 0,
2260             };
2261             next;
2262         }
2263
2264         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2265
2266         if ($dtime > 1000) {
2267             my $dutime = $used -  $old->{used};
2268
2269             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2270             $last_proc_pid_stat->{$pid} = {
2271                 time => $ctime,
2272                 used => $used,
2273                 cpu => $d->{cpu},
2274             };
2275         } else {
2276             $d->{cpu} = $old->{cpu};
2277         }
2278     }
2279
2280     return $res if !$full;
2281
2282     my $qmpclient = PVE::QMPClient->new();
2283
2284     my $ballooncb = sub {
2285         my ($vmid, $resp) = @_;
2286
2287         my $info = $resp->{'return'};
2288         return if !$info->{max_mem};
2289
2290         my $d = $res->{$vmid};
2291
2292         # use memory assigned to VM
2293         $d->{maxmem} = $info->{max_mem};
2294         $d->{balloon} = $info->{actual};
2295
2296         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2297             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2298             $d->{freemem} = $info->{free_mem};
2299         }
2300
2301     };
2302
2303     my $blockstatscb = sub {
2304         my ($vmid, $resp) = @_;
2305         my $data = $resp->{'return'} || [];
2306         my $totalrdbytes = 0;
2307         my $totalwrbytes = 0;
2308         for my $blockstat (@$data) {
2309             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2310             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2311         }
2312         $res->{$vmid}->{diskread} = $totalrdbytes;
2313         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2314     };
2315
2316     my $statuscb = sub {
2317         my ($vmid, $resp) = @_;
2318
2319         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2320         # this fails if ballon driver is not loaded, so this must be
2321         # the last commnand (following command are aborted if this fails).
2322         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2323
2324         my $status = 'unknown';
2325         if (!defined($status = $resp->{'return'}->{status})) {
2326             warn "unable to get VM status\n";
2327             return;
2328         }
2329
2330         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2331     };
2332
2333     foreach my $vmid (keys %$list) {
2334         next if $opt_vmid && ($vmid ne $opt_vmid);
2335         next if !$res->{$vmid}->{pid}; # not running
2336         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2337     }
2338
2339     $qmpclient->queue_execute();
2340
2341     foreach my $vmid (keys %$list) {
2342         next if $opt_vmid && ($vmid ne $opt_vmid);
2343         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2344     }
2345
2346     return $res;
2347 }
2348
2349 sub foreach_drive {
2350     my ($conf, $func) = @_;
2351
2352     foreach my $ds (keys %$conf) {
2353         next if !valid_drivename($ds);
2354
2355         my $drive = parse_drive($ds, $conf->{$ds});
2356         next if !$drive;
2357
2358         &$func($ds, $drive);
2359     }
2360 }
2361
2362 sub foreach_volid {
2363     my ($conf, $func) = @_;
2364
2365     my $volhash = {};
2366
2367     my $test_volid = sub {
2368         my ($volid, $is_cdrom) = @_;
2369
2370         return if !$volid;
2371
2372         $volhash->{$volid} = $is_cdrom || 0;
2373     };
2374
2375     foreach_drive($conf, sub {
2376         my ($ds, $drive) = @_;
2377         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2378     });
2379
2380     foreach my $snapname (keys %{$conf->{snapshots}}) {
2381         my $snap = $conf->{snapshots}->{$snapname};
2382         &$test_volid($snap->{vmstate}, 0);
2383         foreach_drive($snap, sub {
2384             my ($ds, $drive) = @_;
2385             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2386         });
2387     }
2388
2389     foreach my $volid (keys %$volhash) {
2390         &$func($volid, $volhash->{$volid});
2391     }
2392 }
2393
2394 sub vga_conf_has_spice {
2395     my ($vga) = @_;
2396
2397     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2398
2399     return $1 || 1;
2400 }
2401
2402 sub config_to_command {
2403     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2404
2405     my $cmd = [];
2406     my $globalFlags = [];
2407     my $machineFlags = [];
2408     my $rtcFlags = [];
2409     my $cpuFlags = [];
2410     my $devices = [];
2411     my $pciaddr = '';
2412     my $bridges = {};
2413     my $kvmver = kvm_user_version();
2414     my $vernum = 0; # unknown
2415     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2416         $vernum = $1*1000000+$2*1000;
2417     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2418         $vernum = $1*1000000+$2*1000+$3;
2419     }
2420
2421     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2422
2423     my $have_ovz = -f '/proc/vz/vestat';
2424
2425     my $q35 = machine_type_is_q35($conf);
2426
2427     push @$cmd, '/usr/bin/kvm';
2428
2429     push @$cmd, '-id', $vmid;
2430
2431     my $use_virtio = 0;
2432
2433     my $qmpsocket = qmp_socket($vmid);
2434     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2435     push @$cmd, '-mon', "chardev=qmp,mode=control";
2436
2437     my $socket = vnc_socket($vmid);
2438     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2439
2440     push @$cmd, '-pidfile' , pidfile_name($vmid);
2441
2442     push @$cmd, '-daemonize';
2443
2444     if ($conf->{smbios1}) {
2445         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2446     }
2447
2448     push @$cmd, '-object', "iothread,id=iothread0" if $conf->{iothread};
2449
2450     if ($q35) {
2451         # the q35 chipset support native usb2, so we enable usb controller
2452         # by default for this machine type
2453         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2454     } else {
2455         $pciaddr = print_pci_addr("piix3", $bridges);
2456         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2457
2458         my $use_usb2 = 0;
2459         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2460             next if !$conf->{"usb$i"};
2461             $use_usb2 = 1;
2462         }
2463         # include usb device config
2464         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2465     }
2466
2467     my $vga = $conf->{vga};
2468
2469     my $qxlnum = vga_conf_has_spice($vga);
2470     $vga = 'qxl' if $qxlnum;
2471
2472     if (!$vga) {
2473         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2474                                 $conf->{ostype} eq 'win7' ||
2475                                 $conf->{ostype} eq 'w2k8')) {
2476             $vga = 'std';
2477         } else {
2478             $vga = 'cirrus';
2479         }
2480     }
2481
2482     # enable absolute mouse coordinates (needed by vnc)
2483     my $tablet;
2484     if (defined($conf->{tablet})) {
2485         $tablet = $conf->{tablet};
2486     } else {
2487         $tablet = $defaults->{tablet};
2488         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2489         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2490     }
2491
2492     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2493
2494     # host pci devices
2495     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2496         my $d = parse_hostpci($conf->{"hostpci$i"});
2497         next if !$d;
2498
2499         my $pcie = $d->{pcie};
2500         if($pcie){
2501             die "q35 machine model is not enabled" if !$q35;
2502             $pciaddr = print_pcie_addr("hostpci$i");
2503         }else{
2504             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2505         }
2506
2507         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2508         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2509         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2510         if ($xvga && $xvga ne '') {
2511             push @$cpuFlags, 'kvm=off';
2512             $vga = 'none';
2513         }
2514         $driver = "vfio-pci" if $xvga ne '';
2515         my $pcidevices = $d->{pciid};
2516         my $multifunction = 1 if @$pcidevices > 1;
2517
2518         my $j=0;
2519         foreach my $pcidevice (@$pcidevices) {
2520
2521             my $id = "hostpci$i";
2522             $id .= ".$j" if $multifunction;
2523             my $addr = $pciaddr;
2524             $addr .= ".$j" if $multifunction;
2525             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2526
2527             if($j == 0){
2528                 $devicestr .= "$rombar$xvga";
2529                 $devicestr .= ",multifunction=on" if $multifunction;
2530             }
2531
2532             push @$devices, '-device', $devicestr;
2533             $j++;
2534         }
2535     }
2536
2537     # usb devices
2538     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2539         my $d = parse_usb_device($conf->{"usb$i"});
2540         next if !$d;
2541         if ($d->{vendorid} && $d->{productid}) {
2542             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2543         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2544             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2545         } elsif ($d->{spice}) {
2546             # usb redir support for spice
2547             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2548             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2549         }
2550     }
2551
2552     # serial devices
2553     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2554         if (my $path = $conf->{"serial$i"}) {
2555             if ($path eq 'socket') {
2556                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2557                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2558                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2559             } else {
2560                 die "no such serial device\n" if ! -c $path;
2561                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2562                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2563             }
2564         }
2565     }
2566
2567     # parallel devices
2568     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2569         if (my $path = $conf->{"parallel$i"}) {
2570             die "no such parallel device\n" if ! -c $path;
2571             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2572             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2573             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2574         }
2575     }
2576
2577     my $vmname = $conf->{name} || "vm$vmid";
2578
2579     push @$cmd, '-name', $vmname;
2580
2581     my $sockets = 1;
2582     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2583     $sockets = $conf->{sockets} if  $conf->{sockets};
2584
2585     my $cores = $conf->{cores} || 1;
2586     my $maxcpus = $conf->{maxcpus} if $conf->{maxcpus};
2587
2588     if ($maxcpus) {
2589         push @$cmd, '-smp', "cpus=$cores,maxcpus=$maxcpus";
2590     } else {
2591         push @$cmd, '-smp', "sockets=$sockets,cores=$cores";
2592     }
2593
2594     push @$cmd, '-nodefaults';
2595
2596     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2597
2598     my $bootindex_hash = {};
2599     my $i = 1;
2600     foreach my $o (split(//, $bootorder)) {
2601         $bootindex_hash->{$o} = $i*100;
2602         $i++;
2603     }
2604
2605     push @$cmd, '-boot', "menu=on";
2606
2607     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2608
2609     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2610
2611     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2612
2613     # time drift fix
2614     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2615
2616     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2617     my $useLocaltime = $conf->{localtime};
2618
2619     if (my $ost = $conf->{ostype}) {
2620         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2621
2622         if ($ost =~ m/^w/) { # windows
2623             $useLocaltime = 1 if !defined($conf->{localtime});
2624
2625             # use time drift fix when acpi is enabled
2626             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2627                 $tdf = 1 if !defined($conf->{tdf});
2628             }
2629         }
2630
2631         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2632             $ost eq 'wvista') {
2633             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2634             push @$cmd, '-no-hpet';
2635             #push @$cpuFlags , 'hv_vapic" if !$nokvm;  #fixme, my win2008R2 hang at boot with this
2636             push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2637         }
2638
2639         if ($ost eq 'win7' || $ost eq 'win8') {
2640             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2641         }
2642     }
2643
2644     push @$rtcFlags, 'driftfix=slew' if $tdf;
2645
2646     if ($nokvm) {
2647         push @$machineFlags, 'accel=tcg';
2648     } else {
2649         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2650     }
2651
2652     my $machine_type = $forcemachine || $conf->{machine};
2653     if ($machine_type) {
2654         push @$machineFlags, "type=${machine_type}";
2655     }
2656
2657     if ($conf->{startdate}) {
2658         push @$rtcFlags, "base=$conf->{startdate}";
2659     } elsif ($useLocaltime) {
2660         push @$rtcFlags, 'base=localtime';
2661     }
2662
2663     my $cpu = $nokvm ? "qemu64" : "kvm64";
2664     $cpu = $conf->{cpu} if $conf->{cpu};
2665
2666     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2667
2668     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2669
2670     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2671
2672     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2673
2674     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2675
2676     # Note: enforce needs kernel 3.10, so we do not use it for now
2677     # push @$cmd, '-cpu', "$cpu,enforce";
2678     push @$cmd, '-cpu', $cpu;
2679
2680     push @$cmd, '-S' if $conf->{freeze};
2681
2682     # set keyboard layout
2683     my $kb = $conf->{keyboard} || $defaults->{keyboard};
2684     push @$cmd, '-k', $kb if $kb;
2685
2686     # enable sound
2687     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
2688     #push @$cmd, '-soundhw', 'es1370';
2689     #push @$cmd, '-soundhw', $soundhw if $soundhw;
2690
2691     if($conf->{agent}) {
2692         my $qgasocket = qga_socket($vmid);
2693         my $pciaddr = print_pci_addr("qga0", $bridges);
2694         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
2695         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
2696         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
2697     }
2698
2699     my $spice_port;
2700
2701     if ($qxlnum) {
2702         if ($qxlnum > 1) {
2703             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
2704                 for(my $i = 1; $i < $qxlnum; $i++){
2705                     my $pciaddr = print_pci_addr("vga$i", $bridges);
2706                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
2707                 }
2708             } else {
2709                 # assume other OS works like Linux
2710                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
2711                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
2712             }
2713         }
2714
2715         my $pciaddr = print_pci_addr("spice", $bridges);
2716
2717         $spice_port = PVE::Tools::next_spice_port();
2718
2719         push @$devices, '-spice', "tls-port=${spice_port},addr=127.0.0.1,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
2720
2721         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
2722         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
2723         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
2724     }
2725
2726     # enable balloon by default, unless explicitly disabled
2727     if (!defined($conf->{balloon}) || $conf->{balloon}) {
2728         $pciaddr = print_pci_addr("balloon0", $bridges);
2729         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
2730     }
2731
2732     if ($conf->{watchdog}) {
2733         my $wdopts = parse_watchdog($conf->{watchdog});
2734         $pciaddr = print_pci_addr("watchdog", $bridges);
2735         my $watchdog = $wdopts->{model} || 'i6300esb';
2736         push @$devices, '-device', "$watchdog$pciaddr";
2737         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
2738     }
2739
2740     my $vollist = [];
2741     my $scsicontroller = {};
2742     my $ahcicontroller = {};
2743     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
2744
2745     # Add iscsi initiator name if available
2746     if (my $initiator = get_initiator_name()) {
2747         push @$devices, '-iscsi', "initiator-name=$initiator";
2748     }
2749
2750     foreach_drive($conf, sub {
2751         my ($ds, $drive) = @_;
2752
2753         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
2754             push @$vollist, $drive->{file};
2755         }
2756
2757         $use_virtio = 1 if $ds =~ m/^virtio/;
2758
2759         if (drive_is_cdrom ($drive)) {
2760             if ($bootindex_hash->{d}) {
2761                 $drive->{bootindex} = $bootindex_hash->{d};
2762                 $bootindex_hash->{d} += 1;
2763             }
2764         } else {
2765             if ($bootindex_hash->{c}) {
2766                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
2767                 $bootindex_hash->{c} += 1;
2768             }
2769         }
2770
2771         if ($drive->{interface} eq 'scsi') {
2772
2773             my $maxdev = ($scsihw !~ m/^lsi/) ? 256 : 7;
2774             my $controller = int($drive->{index} / $maxdev);
2775             $pciaddr = print_pci_addr("scsihw$controller", $bridges);
2776             push @$devices, '-device', "$scsihw,id=scsihw$controller$pciaddr" if !$scsicontroller->{$controller};
2777             $scsicontroller->{$controller}=1;
2778         }
2779
2780         if ($drive->{interface} eq 'sata') {
2781            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
2782            $pciaddr = print_pci_addr("ahci$controller", $bridges);
2783            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
2784            $ahcicontroller->{$controller}=1;
2785         }
2786
2787         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
2788         push @$devices, '-drive',$drive_cmd;
2789         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
2790     });
2791
2792     push @$cmd, '-m', $conf->{memory} || $defaults->{memory};
2793
2794     for (my $i = 0; $i < $MAX_NETS; $i++) {
2795          next if !$conf->{"net$i"};
2796          my $d = parse_net($conf->{"net$i"});
2797          next if !$d;
2798
2799          $use_virtio = 1 if $d->{model} eq 'virtio';
2800
2801          if ($bootindex_hash->{n}) {
2802             $d->{bootindex} = $bootindex_hash->{n};
2803             $bootindex_hash->{n} += 1;
2804          }
2805
2806          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
2807          push @$devices, '-netdev', $netdevfull;
2808
2809          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
2810          push @$devices, '-device', $netdevicefull;
2811     }
2812
2813     if (!$q35) {
2814         # add pci bridges
2815         while (my ($k, $v) = each %$bridges) {
2816             $pciaddr = print_pci_addr("pci.$k");
2817             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
2818         }
2819     }
2820
2821     # hack: virtio with fairsched is unreliable, so we do not use fairsched
2822     # when the VM uses virtio devices.
2823     if (!$use_virtio && $have_ovz) {
2824
2825         my $cpuunits = defined($conf->{cpuunits}) ?
2826             $conf->{cpuunits} : $defaults->{cpuunits};
2827
2828         push @$cmd, '-cpuunits', $cpuunits if $cpuunits;
2829
2830         # fixme: cpulimit is currently ignored
2831         #push @$cmd, '-cpulimit', $conf->{cpulimit} if $conf->{cpulimit};
2832     }
2833
2834     # add custom args
2835     if ($conf->{args}) {
2836         my $aa = PVE::Tools::split_args($conf->{args});
2837         push @$cmd, @$aa;
2838     }
2839
2840     push @$cmd, @$devices;
2841     push @$cmd, '-rtc', join(',', @$rtcFlags)
2842         if scalar(@$rtcFlags);
2843     push @$cmd, '-machine', join(',', @$machineFlags)
2844         if scalar(@$machineFlags);
2845     push @$cmd, '-global', join(',', @$globalFlags)
2846         if scalar(@$globalFlags);
2847
2848     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
2849 }
2850
2851 sub vnc_socket {
2852     my ($vmid) = @_;
2853     return "${var_run_tmpdir}/$vmid.vnc";
2854 }
2855
2856 sub spice_port {
2857     my ($vmid) = @_;
2858
2859     my $res = vm_mon_cmd($vmid, 'query-spice');
2860
2861     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
2862 }
2863
2864 sub qmp_socket {
2865     my ($vmid) = @_;
2866     return "${var_run_tmpdir}/$vmid.qmp";
2867 }
2868
2869 sub qga_socket {
2870     my ($vmid) = @_;
2871     return "${var_run_tmpdir}/$vmid.qga";
2872 }
2873
2874 sub pidfile_name {
2875     my ($vmid) = @_;
2876     return "${var_run_tmpdir}/$vmid.pid";
2877 }
2878
2879 sub vm_devices_list {
2880     my ($vmid) = @_;
2881
2882     my $res = vm_mon_cmd($vmid, 'query-pci');
2883
2884     my $devices = {};
2885     foreach my $pcibus (@$res) {
2886         foreach my $device (@{$pcibus->{devices}}) {
2887             next if !$device->{'qdev_id'};
2888             $devices->{$device->{'qdev_id'}} = 1;
2889         }
2890     }
2891
2892     my $resblock = vm_mon_cmd($vmid, 'query-block');
2893     foreach my $block (@$resblock) {
2894         if($block->{device} =~ m/^drive-(\S+)/){
2895                 $devices->{$1} = 1;
2896         }
2897     }
2898
2899     return $devices;
2900 }
2901
2902 sub vm_deviceplug {
2903     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
2904
2905     return 1 if !check_running($vmid);
2906
2907     my $q35 = machine_type_is_q35($conf);
2908
2909     if ($deviceid eq 'tablet') {
2910         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
2911         return 1;
2912     }
2913
2914     return 1 if !$conf->{hotplug};
2915
2916     my $devices_list = vm_devices_list($vmid);
2917     return 1 if defined($devices_list->{$deviceid});
2918
2919     qemu_bridgeadd($storecfg, $conf, $vmid, $deviceid); #add bridge if we need it for the device
2920
2921     if ($deviceid =~ m/^(virtio)(\d+)$/) {
2922         return undef if !qemu_driveadd($storecfg, $vmid, $device);
2923         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
2924         qemu_deviceadd($vmid, $devicefull);
2925         if(!qemu_deviceaddverify($vmid, $deviceid)) {
2926            qemu_drivedel($vmid, $deviceid);
2927            return undef;
2928         }
2929     }
2930
2931     if ($deviceid =~ m/^(scsihw)(\d+)$/) {
2932         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
2933         my $pciaddr = print_pci_addr($deviceid);
2934         my $devicefull = "$scsihw,id=$deviceid$pciaddr";
2935         qemu_deviceadd($vmid, $devicefull);
2936         return undef if(!qemu_deviceaddverify($vmid, $deviceid));
2937     }
2938
2939     if ($deviceid =~ m/^(scsi)(\d+)$/) {
2940         return undef if !qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
2941         return undef if !qemu_driveadd($storecfg, $vmid, $device);
2942         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
2943         if(!qemu_deviceadd($vmid, $devicefull)) {
2944            qemu_drivedel($vmid, $deviceid);
2945            return undef;
2946         }
2947     }
2948
2949     if ($deviceid =~ m/^(net)(\d+)$/) {
2950         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
2951         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
2952         qemu_deviceadd($vmid, $netdevicefull);
2953         if(!qemu_deviceaddverify($vmid, $deviceid)) {
2954            qemu_netdevdel($vmid, $deviceid);
2955            return undef;
2956         }
2957     }
2958
2959
2960     if (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
2961         my $bridgeid = $2;
2962         my $pciaddr = print_pci_addr($deviceid);
2963         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
2964         qemu_deviceadd($vmid, $devicefull);
2965         return undef if !qemu_deviceaddverify($vmid, $deviceid);
2966     }
2967
2968     return 1;
2969 }
2970
2971 sub vm_deviceunplug {
2972     my ($vmid, $conf, $deviceid) = @_;
2973
2974     return 1 if !check_running ($vmid);
2975
2976     if ($deviceid eq 'tablet') {
2977         qemu_devicedel($vmid, $deviceid);
2978         return 1;
2979     }
2980
2981     return 1 if !$conf->{hotplug};
2982
2983     my $devices_list = vm_devices_list($vmid);
2984     return 1 if !defined($devices_list->{$deviceid});
2985
2986     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
2987
2988     if ($deviceid =~ m/^(virtio)(\d+)$/) {
2989         qemu_devicedel($vmid, $deviceid);
2990         return undef if !qemu_devicedelverify($vmid, $deviceid);
2991         return undef if !qemu_drivedel($vmid, $deviceid);
2992     }
2993
2994     if ($deviceid =~ m/^(lsi)(\d+)$/) {
2995         return undef if !qemu_devicedel($vmid, $deviceid);
2996     }
2997
2998     if ($deviceid =~ m/^(scsi)(\d+)$/) {
2999         return undef if !qemu_devicedel($vmid, $deviceid);
3000         return undef if !qemu_drivedel($vmid, $deviceid);
3001     }
3002
3003     if ($deviceid =~ m/^(net)(\d+)$/) {
3004         qemu_devicedel($vmid, $deviceid);
3005         return undef if !qemu_devicedelverify($vmid, $deviceid);
3006         return undef if !qemu_netdevdel($vmid, $deviceid);
3007     }
3008
3009     return 1;
3010 }
3011
3012 sub qemu_deviceadd {
3013     my ($vmid, $devicefull) = @_;
3014
3015     $devicefull = "driver=".$devicefull;
3016     my %options =  split(/[=,]/, $devicefull);
3017
3018     vm_mon_cmd($vmid, "device_add" , %options);
3019     return 1;
3020 }
3021
3022 sub qemu_devicedel {
3023     my($vmid, $deviceid) = @_;
3024     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3025     return 1;
3026 }
3027
3028 sub qemu_driveadd {
3029     my($storecfg, $vmid, $device) = @_;
3030
3031     my $drive = print_drive_full($storecfg, $vmid, $device);
3032     my $ret = vm_human_monitor_command($vmid, "drive_add auto $drive");
3033     # If the command succeeds qemu prints: "OK"
3034     if ($ret !~ m/OK/s) {
3035         syslog("err", "adding drive failed: $ret");
3036         return undef;
3037     }
3038     return 1;
3039 }
3040
3041 sub qemu_drivedel {
3042     my($vmid, $deviceid) = @_;
3043
3044     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3045     $ret =~ s/^\s+//;
3046     if ($ret =~ m/Device \'.*?\' not found/s) {
3047         # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3048     }
3049     elsif ($ret ne "") {
3050       syslog("err", "deleting drive $deviceid failed : $ret");
3051       return undef;
3052     }
3053     return 1;
3054 }
3055
3056 sub qemu_deviceaddverify {
3057     my ($vmid,$deviceid) = @_;
3058
3059     for (my $i = 0; $i <= 5; $i++) {
3060          my $devices_list = vm_devices_list($vmid);
3061          return 1 if defined($devices_list->{$deviceid});
3062          sleep 1;
3063     }
3064     syslog("err", "error on hotplug device $deviceid");
3065     return undef;
3066 }
3067
3068
3069 sub qemu_devicedelverify {
3070     my ($vmid,$deviceid) = @_;
3071
3072     #need to verify the device is correctly remove as device_del is async and empty return is not reliable
3073     for (my $i = 0; $i <= 5; $i++) {
3074          my $devices_list = vm_devices_list($vmid);
3075          return 1 if !defined($devices_list->{$deviceid});
3076          sleep 1;
3077     }
3078     syslog("err", "error on hot-unplugging device $deviceid");
3079     return undef;
3080 }
3081
3082 sub qemu_findorcreatescsihw {
3083     my ($storecfg, $conf, $vmid, $device) = @_;
3084
3085     my $maxdev = ($conf->{scsihw} && ($conf->{scsihw} !~ m/^lsi/)) ? 256 : 7;
3086     my $controller = int($device->{index} / $maxdev);
3087     my $scsihwid="scsihw$controller";
3088     my $devices_list = vm_devices_list($vmid);
3089
3090     if(!defined($devices_list->{$scsihwid})) {
3091        return undef if !vm_deviceplug($storecfg, $conf, $vmid, $scsihwid);
3092     }
3093     return 1;
3094 }
3095
3096 sub qemu_bridgeadd {
3097     my ($storecfg, $conf, $vmid, $device) = @_;
3098
3099     my $bridges = {};
3100     my $bridgeid = undef;
3101     print_pci_addr($device, $bridges);
3102
3103     while (my ($k, $v) = each %$bridges) {
3104         $bridgeid = $k;
3105     }
3106     return if !$bridgeid || $bridgeid < 1;
3107     my $bridge = "pci.$bridgeid";
3108     my $devices_list = vm_devices_list($vmid);
3109
3110     if(!defined($devices_list->{$bridge})) {
3111         return undef if !vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3112     }
3113     return 1;
3114 }
3115
3116 sub qemu_netdevadd {
3117     my ($vmid, $conf, $device, $deviceid) = @_;
3118
3119     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3120     my %options =  split(/[=,]/, $netdev);
3121
3122     vm_mon_cmd($vmid, "netdev_add",  %options);
3123     return 1;
3124 }
3125
3126 sub qemu_netdevdel {
3127     my ($vmid, $deviceid) = @_;
3128
3129     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3130     return 1;
3131 }
3132
3133 sub qemu_cpu_hotplug {
3134     my ($vmid, $conf, $cores) = @_;
3135
3136     die "new cores config is not defined" if !$cores;
3137     die "you can't add more cores than maxcpus"
3138         if $conf->{maxcpus} && ($cores > $conf->{maxcpus});
3139     return if !check_running($vmid);
3140
3141     my $currentcores = $conf->{cores} if $conf->{cores};
3142     die "current cores is not defined" if !$currentcores;
3143     die "maxcpus is not defined" if !$conf->{maxcpus};
3144     raise_param_exc({ 'cores' => "online cpu unplug is not yet possible" })
3145         if($cores < $currentcores);
3146
3147     my $currentrunningcores = vm_mon_cmd($vmid, "query-cpus");
3148     raise_param_exc({ 'cores' => "cores number if running vm is different than configuration" })
3149         if scalar (@{$currentrunningcores}) != $currentcores;
3150
3151     for(my $i = $currentcores; $i < $cores; $i++) {
3152         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3153     }
3154 }
3155
3156 sub qemu_block_set_io_throttle {
3157     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3158
3159     return if !check_running($vmid) ;
3160
3161     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3162
3163 }
3164
3165 # old code, only used to shutdown old VM after update
3166 sub __read_avail {
3167     my ($fh, $timeout) = @_;
3168
3169     my $sel = new IO::Select;
3170     $sel->add($fh);
3171
3172     my $res = '';
3173     my $buf;
3174
3175     my @ready;
3176     while (scalar (@ready = $sel->can_read($timeout))) {
3177         my $count;
3178         if ($count = $fh->sysread($buf, 8192)) {
3179             if ($buf =~ /^(.*)\(qemu\) $/s) {
3180                 $res .= $1;
3181                 last;
3182             } else {
3183                 $res .= $buf;
3184             }
3185         } else {
3186             if (!defined($count)) {
3187                 die "$!\n";
3188             }
3189             last;
3190         }
3191     }
3192
3193     die "monitor read timeout\n" if !scalar(@ready);
3194
3195     return $res;
3196 }
3197
3198 # old code, only used to shutdown old VM after update
3199 sub vm_monitor_command {
3200     my ($vmid, $cmdstr, $nocheck) = @_;
3201
3202     my $res;
3203
3204     eval {
3205         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3206
3207         my $sname = "${var_run_tmpdir}/$vmid.mon";
3208
3209         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3210             die "unable to connect to VM $vmid socket - $!\n";
3211
3212         my $timeout = 3;
3213
3214         # hack: migrate sometime blocks the monitor (when migrate_downtime
3215         # is set)
3216         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3217             $timeout = 60*60; # 1 hour
3218         }
3219
3220         # read banner;
3221         my $data = __read_avail($sock, $timeout);
3222
3223         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3224             die "got unexpected qemu monitor banner\n";
3225         }
3226
3227         my $sel = new IO::Select;
3228         $sel->add($sock);
3229
3230         if (!scalar(my @ready = $sel->can_write($timeout))) {
3231             die "monitor write error - timeout";
3232         }
3233
3234         my $fullcmd = "$cmdstr\r";
3235
3236         # syslog('info', "VM $vmid monitor command: $cmdstr");
3237
3238         my $b;
3239         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3240             die "monitor write error - $!";
3241         }
3242
3243         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3244
3245         $timeout = 20;
3246
3247         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3248             $timeout = 60*60; # 1 hour
3249         } elsif ($cmdstr =~ m/^(eject|change)/) {
3250             $timeout = 60; # note: cdrom mount command is slow
3251         }
3252         if ($res = __read_avail($sock, $timeout)) {
3253
3254             my @lines = split("\r?\n", $res);
3255
3256             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3257
3258             $res = join("\n", @lines);
3259             $res .= "\n";
3260         }
3261     };
3262
3263     my $err = $@;
3264
3265     if ($err) {
3266         syslog("err", "VM $vmid monitor command failed - $err");
3267         die $err;
3268     }
3269
3270     return $res;
3271 }
3272
3273 sub qemu_block_resize {
3274     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3275
3276     my $running = check_running($vmid);
3277
3278     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3279
3280     return if !$running;
3281
3282     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3283
3284 }
3285
3286 sub qemu_volume_snapshot {
3287     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3288
3289     my $running = check_running($vmid);
3290
3291     return if !PVE::Storage::volume_snapshot($storecfg, $volid, $snap, $running);
3292
3293     return if !$running;
3294
3295     vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3296
3297 }
3298
3299 sub qemu_volume_snapshot_delete {
3300     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3301
3302     my $running = check_running($vmid);
3303
3304     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3305
3306     return if !$running;
3307
3308     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3309 }
3310
3311 sub qga_freezefs {
3312     my ($vmid) = @_;
3313
3314     #need to impplement call to qemu-ga
3315 }
3316
3317 sub qga_unfreezefs {
3318     my ($vmid) = @_;
3319
3320     #need to impplement call to qemu-ga
3321 }
3322
3323 sub set_migration_caps {
3324     my ($vmid) = @_;
3325
3326     my $cap_ref = [];
3327
3328     my $enabled_cap = {
3329         "auto-converge" => 1,
3330         "xbzrle" => 0,
3331         "x-rdma-pin-all" => 0,
3332         "zero-blocks" => 0,
3333     };
3334
3335     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3336
3337     for my $supported_capability (@$supported_capabilities) {
3338         push @$cap_ref, {
3339             capability => $supported_capability->{capability},
3340             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3341         };
3342     }
3343
3344     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3345 }
3346
3347 sub vm_start {
3348     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
3349
3350     lock_config($vmid, sub {
3351         my $conf = load_config($vmid, $migratedfrom);
3352
3353         die "you can't start a vm if it's a template\n" if is_template($conf);
3354
3355         check_lock($conf) if !$skiplock;
3356
3357         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
3358
3359         my $defaults = load_defaults();
3360
3361         # set environment variable useful inside network script
3362         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
3363
3364         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
3365
3366         my $migrate_port = 0;
3367         my $migrate_uri;
3368         if ($statefile) {
3369             if ($statefile eq 'tcp') {
3370                 my $localip = "localhost";
3371                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
3372                 if ($datacenterconf->{migration_unsecure}) {
3373                         my $nodename = PVE::INotify::nodename();
3374                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
3375                 }
3376                 $migrate_port = PVE::Tools::next_migrate_port();
3377                 $migrate_uri = "tcp:${localip}:${migrate_port}";
3378                 push @$cmd, '-incoming', $migrate_uri;
3379                 push @$cmd, '-S';
3380             } else {
3381                 push @$cmd, '-loadstate', $statefile;
3382             }
3383         } elsif ($paused) {
3384             push @$cmd, '-S';
3385         }
3386
3387         # host pci devices
3388         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3389           my $d = parse_hostpci($conf->{"hostpci$i"});
3390           next if !$d;
3391           my $pcidevices = $d->{pciid};
3392           foreach my $pcidevice (@$pcidevices) {
3393                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
3394
3395                 my $info = pci_device_info("0000:$pciid");
3396                 die "IOMMU not present\n" if !check_iommu_support();
3397                 die "no pci device info for device '$pciid'\n" if !$info;
3398
3399                 if ($d->{driver} && $d->{driver} eq "vfio") {
3400                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
3401                 } else {
3402                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
3403                 }
3404
3405                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
3406           }
3407         }
3408
3409         PVE::Storage::activate_volumes($storecfg, $vollist);
3410
3411         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
3412                     umask => 0077); };
3413         my $err = $@;
3414         die "start failed: $err" if $err;
3415
3416         print "migration listens on $migrate_uri\n" if $migrate_uri;
3417
3418         if ($statefile && $statefile ne 'tcp')  {
3419             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
3420             warn $@ if $@;
3421         }
3422
3423         if ($migratedfrom) {
3424
3425             eval {
3426                 PVE::QemuServer::set_migration_caps($vmid);
3427             };
3428             warn $@ if $@;
3429
3430             if ($spice_port) {
3431                 print "spice listens on port $spice_port\n";
3432                 if ($spice_ticket) {
3433                     PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
3434                     PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
3435                 }
3436             }
3437
3438         } else {
3439
3440             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
3441                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
3442                     if $conf->{balloon};
3443                 vm_mon_cmd_nocheck($vmid, 'qom-set',
3444                             path => "machine/peripheral/balloon0",
3445                             property => "guest-stats-polling-interval",
3446                             value => 2);
3447             }
3448         }
3449     });
3450 }
3451
3452 sub vm_mon_cmd {
3453     my ($vmid, $execute, %params) = @_;
3454
3455     my $cmd = { execute => $execute, arguments => \%params };
3456     vm_qmp_command($vmid, $cmd);
3457 }
3458
3459 sub vm_mon_cmd_nocheck {
3460     my ($vmid, $execute, %params) = @_;
3461
3462     my $cmd = { execute => $execute, arguments => \%params };
3463     vm_qmp_command($vmid, $cmd, 1);
3464 }
3465
3466 sub vm_qmp_command {
3467     my ($vmid, $cmd, $nocheck) = @_;
3468
3469     my $res;
3470
3471     my $timeout;
3472     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
3473         $timeout = $cmd->{arguments}->{timeout};
3474         delete $cmd->{arguments}->{timeout};
3475     }
3476
3477     eval {
3478         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3479         my $sname = qmp_socket($vmid);
3480         if (-e $sname) {
3481             my $qmpclient = PVE::QMPClient->new();
3482
3483             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
3484         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
3485             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
3486                 if scalar(%{$cmd->{arguments}});
3487             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
3488         } else {
3489             die "unable to open monitor socket\n";
3490         }
3491     };
3492     if (my $err = $@) {
3493         syslog("err", "VM $vmid qmp command failed - $err");
3494         die $err;
3495     }
3496
3497     return $res;
3498 }
3499
3500 sub vm_human_monitor_command {
3501     my ($vmid, $cmdline) = @_;
3502
3503     my $res;
3504
3505     my $cmd = {
3506         execute => 'human-monitor-command',
3507         arguments => { 'command-line' => $cmdline},
3508     };
3509
3510     return vm_qmp_command($vmid, $cmd);
3511 }
3512
3513 sub vm_commandline {
3514     my ($storecfg, $vmid) = @_;
3515
3516     my $conf = load_config($vmid);
3517
3518     my $defaults = load_defaults();
3519
3520     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
3521
3522     return join(' ', @$cmd);
3523 }
3524
3525 sub vm_reset {
3526     my ($vmid, $skiplock) = @_;
3527
3528     lock_config($vmid, sub {
3529
3530         my $conf = load_config($vmid);
3531
3532         check_lock($conf) if !$skiplock;
3533
3534         vm_mon_cmd($vmid, "system_reset");
3535     });
3536 }
3537
3538 sub get_vm_volumes {
3539     my ($conf) = @_;
3540
3541     my $vollist = [];
3542     foreach_volid($conf, sub {
3543         my ($volid, $is_cdrom) = @_;
3544
3545         return if $volid =~ m|^/|;
3546
3547         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3548         return if !$sid;
3549
3550         push @$vollist, $volid;
3551     });
3552
3553     return $vollist;
3554 }
3555
3556 sub vm_stop_cleanup {
3557     my ($storecfg, $vmid, $conf, $keepActive) = @_;
3558
3559     eval {
3560         fairsched_rmnod($vmid); # try to destroy group
3561
3562         if (!$keepActive) {
3563             my $vollist = get_vm_volumes($conf);
3564             PVE::Storage::deactivate_volumes($storecfg, $vollist);
3565         }
3566
3567         foreach my $ext (qw(mon qmp pid vnc qga)) {
3568             unlink "/var/run/qemu-server/${vmid}.$ext";
3569         }
3570     };
3571     warn $@ if $@; # avoid errors - just warn
3572 }
3573
3574 # Note: use $nockeck to skip tests if VM configuration file exists.
3575 # We need that when migration VMs to other nodes (files already moved)
3576 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
3577 sub vm_stop {
3578     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
3579
3580     $force = 1 if !defined($force) && !$shutdown;
3581
3582     if ($migratedfrom){
3583         my $pid = check_running($vmid, $nocheck, $migratedfrom);
3584         kill 15, $pid if $pid;
3585         my $conf = load_config($vmid, $migratedfrom);
3586         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive);
3587         return;
3588     }
3589
3590     lock_config($vmid, sub {
3591
3592         my $pid = check_running($vmid, $nocheck);
3593         return if !$pid;
3594
3595         my $conf;
3596         if (!$nocheck) {
3597             $conf = load_config($vmid);
3598             check_lock($conf) if !$skiplock;
3599             if (!defined($timeout) && $shutdown && $conf->{startup}) {
3600                 my $opts = parse_startup($conf->{startup});
3601                 $timeout = $opts->{down} if $opts->{down};
3602             }
3603         }
3604
3605         $timeout = 60 if !defined($timeout);
3606
3607         eval {
3608             if ($shutdown) {
3609                 $nocheck ? vm_mon_cmd_nocheck($vmid, "system_powerdown") : vm_mon_cmd($vmid, "system_powerdown");
3610
3611             } else {
3612                 $nocheck ? vm_mon_cmd_nocheck($vmid, "quit") : vm_mon_cmd($vmid, "quit");
3613             }
3614         };
3615         my $err = $@;
3616
3617         if (!$err) {
3618             my $count = 0;
3619             while (($count < $timeout) && check_running($vmid, $nocheck)) {
3620                 $count++;
3621                 sleep 1;
3622             }
3623
3624             if ($count >= $timeout) {
3625                 if ($force) {
3626                     warn "VM still running - terminating now with SIGTERM\n";
3627                     kill 15, $pid;
3628                 } else {
3629                     die "VM quit/powerdown failed - got timeout\n";
3630                 }
3631             } else {
3632                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive) if $conf;
3633                 return;
3634             }
3635         } else {
3636             if ($force) {
3637                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
3638                 kill 15, $pid;
3639             } else {
3640                 die "VM quit/powerdown failed\n";
3641             }
3642         }
3643
3644         # wait again
3645         $timeout = 10;
3646
3647         my $count = 0;
3648         while (($count < $timeout) && check_running($vmid, $nocheck)) {
3649             $count++;
3650             sleep 1;
3651         }
3652
3653         if ($count >= $timeout) {
3654             warn "VM still running - terminating now with SIGKILL\n";
3655             kill 9, $pid;
3656             sleep 1;
3657         }
3658
3659         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive) if $conf;
3660    });
3661 }
3662
3663 sub vm_suspend {
3664     my ($vmid, $skiplock) = @_;
3665
3666     lock_config($vmid, sub {
3667
3668         my $conf = load_config($vmid);
3669
3670         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
3671
3672         vm_mon_cmd($vmid, "stop");
3673     });
3674 }
3675
3676 sub vm_resume {
3677     my ($vmid, $skiplock) = @_;
3678
3679     lock_config($vmid, sub {
3680
3681         my $conf = load_config($vmid);
3682
3683         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
3684
3685         vm_mon_cmd($vmid, "cont");
3686     });
3687 }
3688
3689 sub vm_sendkey {
3690     my ($vmid, $skiplock, $key) = @_;
3691
3692     lock_config($vmid, sub {
3693
3694         my $conf = load_config($vmid);
3695
3696         # there is no qmp command, so we use the human monitor command
3697         vm_human_monitor_command($vmid, "sendkey $key");
3698     });
3699 }
3700
3701 sub vm_destroy {
3702     my ($storecfg, $vmid, $skiplock) = @_;
3703
3704     lock_config($vmid, sub {
3705
3706         my $conf = load_config($vmid);
3707
3708         check_lock($conf) if !$skiplock;
3709
3710         if (!check_running($vmid)) {
3711             fairsched_rmnod($vmid); # try to destroy group
3712             destroy_vm($storecfg, $vmid);
3713         } else {
3714             die "VM $vmid is running - destroy failed\n";
3715         }
3716     });
3717 }
3718
3719 # pci helpers
3720
3721 sub file_write {
3722     my ($filename, $buf) = @_;
3723
3724     my $fh = IO::File->new($filename, "w");
3725     return undef if !$fh;
3726
3727     my $res = print $fh $buf;
3728
3729     $fh->close();
3730
3731     return $res;
3732 }
3733
3734 sub pci_device_info {
3735     my ($name) = @_;
3736
3737     my $res;
3738
3739     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
3740     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
3741
3742     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
3743     return undef if !defined($irq) || $irq !~ m/^\d+$/;
3744
3745     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
3746     return undef if !defined($vendor) || $vendor !~ s/^0x//;
3747
3748     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
3749     return undef if !defined($product) || $product !~ s/^0x//;
3750
3751     $res = {
3752         name => $name,
3753         vendor => $vendor,
3754         product => $product,
3755         domain => $domain,
3756         bus => $bus,
3757         slot => $slot,
3758         func => $func,
3759         irq => $irq,
3760         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
3761     };
3762
3763     return $res;
3764 }
3765
3766 sub pci_dev_reset {
3767     my ($dev) = @_;
3768
3769     my $name = $dev->{name};
3770
3771     my $fn = "$pcisysfs/devices/$name/reset";
3772
3773     return file_write($fn, "1");
3774 }
3775
3776 sub pci_dev_bind_to_stub {
3777     my ($dev) = @_;
3778
3779     my $name = $dev->{name};
3780
3781     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
3782     return 1 if -d $testdir;
3783
3784     my $data = "$dev->{vendor} $dev->{product}";
3785     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
3786
3787     my $fn = "$pcisysfs/devices/$name/driver/unbind";
3788     if (!file_write($fn, $name)) {
3789         return undef if -f $fn;
3790     }
3791
3792     $fn = "$pcisysfs/drivers/pci-stub/bind";
3793     if (! -d $testdir) {
3794         return undef if !file_write($fn, $name);
3795     }
3796
3797     return -d $testdir;
3798 }
3799
3800 sub pci_dev_bind_to_vfio {
3801     my ($dev) = @_;
3802
3803     my $name = $dev->{name};
3804
3805     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
3806
3807     if (!-d $vfio_basedir) {
3808         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
3809     }
3810     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
3811
3812     my $testdir = "$vfio_basedir/$name";
3813     return 1 if -d $testdir;
3814
3815     my $data = "$dev->{vendor} $dev->{product}";
3816     return undef if !file_write("$vfio_basedir/new_id", $data);
3817
3818     my $fn = "$pcisysfs/devices/$name/driver/unbind";
3819     if (!file_write($fn, $name)) {
3820         return undef if -f $fn;
3821     }
3822
3823     $fn = "$vfio_basedir/bind";
3824     if (! -d $testdir) {
3825         return undef if !file_write($fn, $name);
3826     }
3827
3828     return -d $testdir;
3829 }
3830
3831 sub pci_dev_group_bind_to_vfio {
3832     my ($pciid) = @_;
3833
3834     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
3835
3836     if (!-d $vfio_basedir) {
3837         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
3838     }
3839     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
3840
3841     # get IOMMU group devices
3842     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
3843       my @devs = grep /^0000:/, readdir($D);
3844     closedir($D);
3845
3846     foreach my $pciid (@devs) {
3847         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
3848         my $info = pci_device_info($1);
3849         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
3850     }
3851
3852     return 1;
3853 }
3854
3855 sub print_pci_addr {
3856     my ($id, $bridges) = @_;
3857
3858     my $res = '';
3859     my $devices = {
3860         piix3 => { bus => 0, addr => 1 },
3861         #addr2 : first videocard
3862         balloon0 => { bus => 0, addr => 3 },
3863         watchdog => { bus => 0, addr => 4 },
3864         scsihw0 => { bus => 0, addr => 5 },
3865         scsihw1 => { bus => 0, addr => 6 },
3866         ahci0 => { bus => 0, addr => 7 },
3867         qga0 => { bus => 0, addr => 8 },
3868         spice => { bus => 0, addr => 9 },
3869         virtio0 => { bus => 0, addr => 10 },
3870         virtio1 => { bus => 0, addr => 11 },
3871         virtio2 => { bus => 0, addr => 12 },
3872         virtio3 => { bus => 0, addr => 13 },
3873         virtio4 => { bus => 0, addr => 14 },
3874         virtio5 => { bus => 0, addr => 15 },
3875         hostpci0 => { bus => 0, addr => 16 },
3876         hostpci1 => { bus => 0, addr => 17 },
3877         net0 => { bus => 0, addr => 18 },
3878         net1 => { bus => 0, addr => 19 },
3879         net2 => { bus => 0, addr => 20 },
3880         net3 => { bus => 0, addr => 21 },
3881         net4 => { bus => 0, addr => 22 },
3882         net5 => { bus => 0, addr => 23 },
3883         vga1 => { bus => 0, addr => 24 },
3884         vga2 => { bus => 0, addr => 25 },
3885         vga3 => { bus => 0, addr => 26 },
3886         hostpci2 => { bus => 0, addr => 27 },
3887         hostpci3 => { bus => 0, addr => 28 },
3888         #addr29 : usb-host (pve-usb.cfg)
3889         'pci.1' => { bus => 0, addr => 30 },
3890         'pci.2' => { bus => 0, addr => 31 },
3891         'net6' => { bus => 1, addr => 1 },
3892         'net7' => { bus => 1, addr => 2 },
3893         'net8' => { bus => 1, addr => 3 },
3894         'net9' => { bus => 1, addr => 4 },
3895         'net10' => { bus => 1, addr => 5 },
3896         'net11' => { bus => 1, addr => 6 },
3897         'net12' => { bus => 1, addr => 7 },
3898         'net13' => { bus => 1, addr => 8 },
3899         'net14' => { bus => 1, addr => 9 },
3900         'net15' => { bus => 1, addr => 10 },
3901         'net16' => { bus => 1, addr => 11 },
3902         'net17' => { bus => 1, addr => 12 },
3903         'net18' => { bus => 1, addr => 13 },
3904         'net19' => { bus => 1, addr => 14 },
3905         'net20' => { bus => 1, addr => 15 },
3906         'net21' => { bus => 1, addr => 16 },
3907         'net22' => { bus => 1, addr => 17 },
3908         'net23' => { bus => 1, addr => 18 },
3909         'net24' => { bus => 1, addr => 19 },
3910         'net25' => { bus => 1, addr => 20 },
3911         'net26' => { bus => 1, addr => 21 },
3912         'net27' => { bus => 1, addr => 22 },
3913         'net28' => { bus => 1, addr => 23 },
3914         'net29' => { bus => 1, addr => 24 },
3915         'net30' => { bus => 1, addr => 25 },
3916         'net31' => { bus => 1, addr => 26 },
3917         'virtio6' => { bus => 2, addr => 1 },
3918         'virtio7' => { bus => 2, addr => 2 },
3919         'virtio8' => { bus => 2, addr => 3 },
3920         'virtio9' => { bus => 2, addr => 4 },
3921         'virtio10' => { bus => 2, addr => 5 },
3922         'virtio11' => { bus => 2, addr => 6 },
3923         'virtio12' => { bus => 2, addr => 7 },
3924         'virtio13' => { bus => 2, addr => 8 },
3925         'virtio14' => { bus => 2, addr => 9 },
3926         'virtio15' => { bus => 2, addr => 10 },
3927     };
3928
3929     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
3930            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
3931            my $bus = $devices->{$id}->{bus};
3932            $res = ",bus=pci.$bus,addr=$addr";
3933            $bridges->{$bus} = 1 if $bridges;
3934     }
3935     return $res;
3936
3937 }
3938
3939 sub print_pcie_addr {
3940     my ($id) = @_;
3941
3942     my $res = '';
3943     my $devices = {
3944         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
3945         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
3946         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
3947         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
3948     };
3949
3950     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
3951            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
3952            my $bus = $devices->{$id}->{bus};
3953            $res = ",bus=$bus,addr=$addr";
3954     }
3955     return $res;
3956
3957 }
3958
3959 # vzdump restore implementaion
3960
3961 sub tar_archive_read_firstfile {
3962     my $archive = shift;
3963
3964     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
3965
3966     # try to detect archive type first
3967     my $pid = open (TMP, "tar tf '$archive'|") ||
3968         die "unable to open file '$archive'\n";
3969     my $firstfile = <TMP>;
3970     kill 15, $pid;
3971     close TMP;
3972
3973     die "ERROR: archive contaions no data\n" if !$firstfile;
3974     chomp $firstfile;
3975
3976     return $firstfile;
3977 }
3978
3979 sub tar_restore_cleanup {
3980     my ($storecfg, $statfile) = @_;
3981
3982     print STDERR "starting cleanup\n";
3983
3984     if (my $fd = IO::File->new($statfile, "r")) {
3985         while (defined(my $line = <$fd>)) {
3986             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
3987                 my $volid = $2;
3988                 eval {
3989                     if ($volid =~ m|^/|) {
3990                         unlink $volid || die 'unlink failed\n';
3991                     } else {
3992                         PVE::Storage::vdisk_free($storecfg, $volid);
3993                     }
3994                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
3995                 };
3996                 print STDERR "unable to cleanup '$volid' - $@" if $@;
3997             } else {
3998                 print STDERR "unable to parse line in statfile - $line";
3999             }
4000         }
4001         $fd->close();
4002     }
4003 }
4004
4005 sub restore_archive {
4006     my ($archive, $vmid, $user, $opts) = @_;
4007
4008     my $format = $opts->{format};
4009     my $comp;
4010
4011     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4012         $format = 'tar' if !$format;
4013         $comp = 'gzip';
4014     } elsif ($archive =~ m/\.tar$/) {
4015         $format = 'tar' if !$format;
4016     } elsif ($archive =~ m/.tar.lzo$/) {
4017         $format = 'tar' if !$format;
4018         $comp = 'lzop';
4019     } elsif ($archive =~ m/\.vma$/) {
4020         $format = 'vma' if !$format;
4021     } elsif ($archive =~ m/\.vma\.gz$/) {
4022         $format = 'vma' if !$format;
4023         $comp = 'gzip';
4024     } elsif ($archive =~ m/\.vma\.lzo$/) {
4025         $format = 'vma' if !$format;
4026         $comp = 'lzop';
4027     } else {
4028         $format = 'vma' if !$format; # default
4029     }
4030
4031     # try to detect archive format
4032     if ($format eq 'tar') {
4033         return restore_tar_archive($archive, $vmid, $user, $opts);
4034     } else {
4035         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4036     }
4037 }
4038
4039 sub restore_update_config_line {
4040     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4041
4042     return if $line =~ m/^\#qmdump\#/;
4043     return if $line =~ m/^\#vzdump\#/;
4044     return if $line =~ m/^lock:/;
4045     return if $line =~ m/^unused\d+:/;
4046     return if $line =~ m/^parent:/;
4047     return if $line =~ m/^template:/; # restored VM is never a template
4048
4049     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4050         # try to convert old 1.X settings
4051         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4052         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4053             my ($model, $macaddr) = split(/\=/, $devconfig);
4054             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4055             my $net = {
4056                 model => $model,
4057                 bridge => "vmbr$ind",
4058                 macaddr => $macaddr,
4059             };
4060             my $netstr = print_net($net);
4061
4062             print $outfd "net$cookie->{netcount}: $netstr\n";
4063             $cookie->{netcount}++;
4064         }
4065     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4066         my ($id, $netstr) = ($1, $2);
4067         my $net = parse_net($netstr);
4068         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4069         $netstr = print_net($net);
4070         print $outfd "$id: $netstr\n";
4071     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4072         my $virtdev = $1;
4073         my $value = $3;
4074         if ($line =~ m/backup=no/) {
4075             print $outfd "#$line";
4076         } elsif ($virtdev && $map->{$virtdev}) {
4077             my $di = parse_drive($virtdev, $value);
4078             delete $di->{format}; # format can change on restore
4079             $di->{file} = $map->{$virtdev};
4080             $value = print_drive($vmid, $di);
4081             print $outfd "$virtdev: $value\n";
4082         } else {
4083             print $outfd $line;
4084         }
4085     } else {
4086         print $outfd $line;
4087     }
4088 }
4089
4090 sub scan_volids {
4091     my ($cfg, $vmid) = @_;
4092
4093     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4094
4095     my $volid_hash = {};
4096     foreach my $storeid (keys %$info) {
4097         foreach my $item (@{$info->{$storeid}}) {
4098             next if !($item->{volid} && $item->{size});
4099             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4100             $volid_hash->{$item->{volid}} = $item;
4101         }
4102     }
4103
4104     return $volid_hash;
4105 }
4106
4107 sub get_used_paths {
4108     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
4109
4110     my $used_path = {};
4111
4112     my $scan_config = sub {
4113         my ($cref, $snapname) = @_;
4114
4115         foreach my $key (keys %$cref) {
4116             my $value = $cref->{$key};
4117             if (valid_drivename($key)) {
4118                 next if $skip_drive && $key eq $skip_drive;
4119                 my $drive = parse_drive($key, $value);
4120                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
4121                 if ($drive->{file} =~ m!^/!) {
4122                     $used_path->{$drive->{file}}++; # = 1;
4123                 } else {
4124                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
4125                     next if !$storeid;
4126                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
4127                     next if !$scfg;
4128                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
4129                     $used_path->{$path}++; # = 1;
4130                 }
4131             }
4132         }
4133     };
4134
4135     &$scan_config($conf);
4136
4137     undef $skip_drive;
4138
4139     if ($scan_snapshots) {
4140         foreach my $snapname (keys %{$conf->{snapshots}}) {
4141             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
4142         }
4143     }
4144
4145     return $used_path;
4146 }
4147
4148 sub update_disksize {
4149     my ($vmid, $conf, $volid_hash) = @_;
4150
4151     my $changes;
4152
4153     my $used = {};
4154
4155     # Note: it is allowed to define multiple storages with same path (alias), so
4156     # we need to check both 'volid' and real 'path' (two different volid can point
4157     # to the same path).
4158
4159     my $usedpath = {};
4160
4161     # update size info
4162     foreach my $opt (keys %$conf) {
4163         if (valid_drivename($opt)) {
4164             my $drive = parse_drive($opt, $conf->{$opt});
4165             my $volid = $drive->{file};
4166             next if !$volid;
4167
4168             $used->{$volid} = 1;
4169             if ($volid_hash->{$volid} &&
4170                 (my $path = $volid_hash->{$volid}->{path})) {
4171                 $usedpath->{$path} = 1;
4172             }
4173
4174             next if drive_is_cdrom($drive);
4175             next if !$volid_hash->{$volid};
4176
4177             $drive->{size} = $volid_hash->{$volid}->{size};
4178             my $new = print_drive($vmid, $drive);
4179             if ($new ne $conf->{$opt}) {
4180                 $changes = 1;
4181                 $conf->{$opt} = $new;
4182             }
4183         }
4184     }
4185
4186     # remove 'unusedX' entry if volume is used
4187     foreach my $opt (keys %$conf) {
4188         next if $opt !~ m/^unused\d+$/;
4189         my $volid = $conf->{$opt};
4190         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
4191         if ($used->{$volid} || ($path && $usedpath->{$path})) {
4192             $changes = 1;
4193             delete $conf->{$opt};
4194         }
4195     }
4196
4197     foreach my $volid (sort keys %$volid_hash) {
4198         next if $volid =~ m/vm-$vmid-state-/;
4199         next if $used->{$volid};
4200         my $path = $volid_hash->{$volid}->{path};
4201         next if !$path; # just to be sure
4202         next if $usedpath->{$path};
4203         $changes = 1;
4204         add_unused_volume($conf, $volid);
4205         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
4206     }
4207
4208     return $changes;
4209 }
4210
4211 sub rescan {
4212     my ($vmid, $nolock) = @_;
4213
4214     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
4215
4216     my $volid_hash = scan_volids($cfg, $vmid);
4217
4218     my $updatefn =  sub {
4219         my ($vmid) = @_;
4220
4221         my $conf = load_config($vmid);
4222
4223         check_lock($conf);
4224
4225         my $vm_volids = {};
4226         foreach my $volid (keys %$volid_hash) {
4227             my $info = $volid_hash->{$volid};
4228             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
4229         }
4230
4231         my $changes = update_disksize($vmid, $conf, $vm_volids);
4232
4233         update_config_nolock($vmid, $conf, 1) if $changes;
4234     };
4235
4236     if (defined($vmid)) {
4237         if ($nolock) {
4238             &$updatefn($vmid);
4239         } else {
4240             lock_config($vmid, $updatefn, $vmid);
4241         }
4242     } else {
4243         my $vmlist = config_list();
4244         foreach my $vmid (keys %$vmlist) {
4245             if ($nolock) {
4246                 &$updatefn($vmid);
4247             } else {
4248                 lock_config($vmid, $updatefn, $vmid);
4249             }
4250         }
4251     }
4252 }
4253
4254 sub restore_vma_archive {
4255     my ($archive, $vmid, $user, $opts, $comp) = @_;
4256
4257     my $input = $archive eq '-' ? "<&STDIN" : undef;
4258     my $readfrom = $archive;
4259
4260     my $uncomp = '';
4261     if ($comp) {
4262         $readfrom = '-';
4263         my $qarchive = PVE::Tools::shellquote($archive);
4264         if ($comp eq 'gzip') {
4265             $uncomp = "zcat $qarchive|";
4266         } elsif ($comp eq 'lzop') {
4267             $uncomp = "lzop -d -c $qarchive|";
4268         } else {
4269             die "unknown compression method '$comp'\n";
4270         }
4271
4272     }
4273
4274     my $tmpdir = "/var/tmp/vzdumptmp$$";
4275     rmtree $tmpdir;
4276
4277     # disable interrupts (always do cleanups)
4278     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
4279         warn "got interrupt - ignored\n";
4280     };
4281
4282     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
4283     POSIX::mkfifo($mapfifo, 0600);
4284     my $fifofh;
4285
4286     my $openfifo = sub {
4287         open($fifofh, '>', $mapfifo) || die $!;
4288     };
4289
4290     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
4291
4292     my $oldtimeout;
4293     my $timeout = 5;
4294
4295     my $devinfo = {};
4296
4297     my $rpcenv = PVE::RPCEnvironment::get();
4298
4299     my $conffile = config_file($vmid);
4300     my $tmpfn = "$conffile.$$.tmp";
4301
4302     # Note: $oldconf is undef if VM does not exists
4303     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
4304
4305     my $print_devmap = sub {
4306         my $virtdev_hash = {};
4307
4308         my $cfgfn = "$tmpdir/qemu-server.conf";
4309
4310         # we can read the config - that is already extracted
4311         my $fh = IO::File->new($cfgfn, "r") ||
4312             "unable to read qemu-server.conf - $!\n";
4313
4314         while (defined(my $line = <$fh>)) {
4315             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
4316                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
4317                 die "archive does not contain data for drive '$virtdev'\n"
4318                     if !$devinfo->{$devname};
4319                 if (defined($opts->{storage})) {
4320                     $storeid = $opts->{storage} || 'local';
4321                 } elsif (!$storeid) {
4322                     $storeid = 'local';
4323                 }
4324                 $format = 'raw' if !$format;
4325                 $devinfo->{$devname}->{devname} = $devname;
4326                 $devinfo->{$devname}->{virtdev} = $virtdev;
4327                 $devinfo->{$devname}->{format} = $format;
4328                 $devinfo->{$devname}->{storeid} = $storeid;
4329
4330                 # check permission on storage
4331                 my $pool = $opts->{pool}; # todo: do we need that?
4332                 if ($user ne 'root@pam') {
4333                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
4334                 }
4335
4336                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
4337             }
4338         }
4339
4340         foreach my $devname (keys %$devinfo) {
4341             die "found no device mapping information for device '$devname'\n"
4342                 if !$devinfo->{$devname}->{virtdev};
4343         }
4344
4345         my $cfg = cfs_read_file('storage.cfg');
4346
4347         # create empty/temp config
4348         if ($oldconf) {
4349             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
4350             foreach_drive($oldconf, sub {
4351                 my ($ds, $drive) = @_;
4352
4353                 return if drive_is_cdrom($drive);
4354
4355                 my $volid = $drive->{file};
4356
4357                 return if !$volid || $volid =~ m|^/|;
4358
4359                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
4360                 return if !$path || !$owner || ($owner != $vmid);
4361
4362                 # Note: only delete disk we want to restore
4363                 # other volumes will become unused
4364                 if ($virtdev_hash->{$ds}) {
4365                     PVE::Storage::vdisk_free($cfg, $volid);
4366                 }
4367             });
4368         }
4369
4370         my $map = {};
4371         foreach my $virtdev (sort keys %$virtdev_hash) {
4372             my $d = $virtdev_hash->{$virtdev};
4373             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
4374             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
4375
4376             # test if requested format is supported
4377             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
4378             my $supported = grep { $_ eq $d->{format} } @$validFormats;
4379             $d->{format} = $defFormat if !$supported;
4380
4381             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
4382                                                   $d->{format}, undef, $alloc_size);
4383             print STDERR "new volume ID is '$volid'\n";
4384             $d->{volid} = $volid;
4385             my $path = PVE::Storage::path($cfg, $volid);
4386
4387             my $write_zeros = 1;
4388             # fixme: what other storages types initialize volumes with zero?
4389             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
4390                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
4391                 $write_zeros = 0;
4392             }
4393
4394             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
4395
4396             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
4397             $map->{$virtdev} = $volid;
4398         }
4399
4400         $fh->seek(0, 0) || die "seek failed - $!\n";
4401
4402         my $outfd = new IO::File ($tmpfn, "w") ||
4403             die "unable to write config for VM $vmid\n";
4404
4405         my $cookie = { netcount => 0 };
4406         while (defined(my $line = <$fh>)) {
4407             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
4408         }
4409
4410         $fh->close();
4411         $outfd->close();
4412     };
4413
4414     eval {
4415         # enable interrupts
4416         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
4417             die "interrupted by signal\n";
4418         };
4419         local $SIG{ALRM} = sub { die "got timeout\n"; };
4420
4421         $oldtimeout = alarm($timeout);
4422
4423         my $parser = sub {
4424             my $line = shift;
4425
4426             print "$line\n";
4427
4428             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
4429                 my ($dev_id, $size, $devname) = ($1, $2, $3);
4430                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
4431             } elsif ($line =~ m/^CTIME: /) {
4432                 # we correctly received the vma config, so we can disable
4433                 # the timeout now for disk allocation (set to 10 minutes, so
4434                 # that we always timeout if something goes wrong)
4435                 alarm(600);
4436                 &$print_devmap();
4437                 print $fifofh "done\n";
4438                 my $tmp = $oldtimeout || 0;
4439                 $oldtimeout = undef;
4440                 alarm($tmp);
4441                 close($fifofh);
4442             }
4443         };
4444
4445         print "restore vma archive: $cmd\n";
4446         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
4447     };
4448     my $err = $@;
4449
4450     alarm($oldtimeout) if $oldtimeout;
4451
4452     unlink $mapfifo;
4453
4454     if ($err) {
4455         rmtree $tmpdir;
4456         unlink $tmpfn;
4457
4458         my $cfg = cfs_read_file('storage.cfg');
4459         foreach my $devname (keys %$devinfo) {
4460             my $volid = $devinfo->{$devname}->{volid};
4461             next if !$volid;
4462             eval {
4463                 if ($volid =~ m|^/|) {
4464                     unlink $volid || die 'unlink failed\n';
4465                 } else {
4466                     PVE::Storage::vdisk_free($cfg, $volid);
4467                 }
4468                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
4469             };
4470             print STDERR "unable to cleanup '$volid' - $@" if $@;
4471         }
4472         die $err;
4473     }
4474
4475     rmtree $tmpdir;
4476
4477     rename($tmpfn, $conffile) ||
4478         die "unable to commit configuration file '$conffile'\n";
4479
4480     PVE::Cluster::cfs_update(); # make sure we read new file
4481
4482     eval { rescan($vmid, 1); };
4483     warn $@ if $@;
4484 }
4485
4486 sub restore_tar_archive {
4487     my ($archive, $vmid, $user, $opts) = @_;
4488
4489     if ($archive ne '-') {
4490         my $firstfile = tar_archive_read_firstfile($archive);
4491         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
4492             if $firstfile ne 'qemu-server.conf';
4493     }
4494
4495     my $storecfg = cfs_read_file('storage.cfg');
4496
4497     # destroy existing data - keep empty config
4498     my $vmcfgfn = PVE::QemuServer::config_file($vmid);
4499     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
4500
4501     my $tocmd = "/usr/lib/qemu-server/qmextract";
4502
4503     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
4504     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
4505     $tocmd .= ' --prealloc' if $opts->{prealloc};
4506     $tocmd .= ' --info' if $opts->{info};
4507
4508     # tar option "xf" does not autodetect compression when read from STDIN,
4509     # so we pipe to zcat
4510     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
4511         PVE::Tools::shellquote("--to-command=$tocmd");
4512
4513     my $tmpdir = "/var/tmp/vzdumptmp$$";
4514     mkpath $tmpdir;
4515
4516     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
4517     local $ENV{VZDUMP_VMID} = $vmid;
4518     local $ENV{VZDUMP_USER} = $user;
4519
4520     my $conffile = config_file($vmid);
4521     my $tmpfn = "$conffile.$$.tmp";
4522
4523     # disable interrupts (always do cleanups)
4524     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
4525         print STDERR "got interrupt - ignored\n";
4526     };
4527
4528     eval {
4529         # enable interrupts
4530         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
4531             die "interrupted by signal\n";
4532         };
4533
4534         if ($archive eq '-') {
4535             print "extracting archive from STDIN\n";
4536             run_command($cmd, input => "<&STDIN");
4537         } else {
4538             print "extracting archive '$archive'\n";
4539             run_command($cmd);
4540         }
4541
4542         return if $opts->{info};
4543
4544         # read new mapping
4545         my $map = {};
4546         my $statfile = "$tmpdir/qmrestore.stat";
4547         if (my $fd = IO::File->new($statfile, "r")) {
4548             while (defined (my $line = <$fd>)) {
4549                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4550                     $map->{$1} = $2 if $1;
4551                 } else {
4552                     print STDERR "unable to parse line in statfile - $line\n";
4553                 }
4554             }
4555             $fd->close();
4556         }
4557
4558         my $confsrc = "$tmpdir/qemu-server.conf";
4559
4560         my $srcfd = new IO::File($confsrc, "r") ||
4561             die "unable to open file '$confsrc'\n";
4562
4563         my $outfd = new IO::File ($tmpfn, "w") ||
4564             die "unable to write config for VM $vmid\n";
4565
4566         my $cookie = { netcount => 0 };
4567         while (defined (my $line = <$srcfd>)) {
4568             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
4569         }
4570
4571         $srcfd->close();
4572         $outfd->close();
4573     };
4574     my $err = $@;
4575
4576     if ($err) {
4577
4578         unlink $tmpfn;
4579
4580         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
4581
4582         die $err;
4583     }
4584
4585     rmtree $tmpdir;
4586
4587     rename $tmpfn, $conffile ||
4588         die "unable to commit configuration file '$conffile'\n";
4589
4590     PVE::Cluster::cfs_update(); # make sure we read new file
4591
4592     eval { rescan($vmid, 1); };
4593     warn $@ if $@;
4594 };
4595
4596
4597 # Internal snapshots
4598
4599 # NOTE: Snapshot create/delete involves several non-atomic
4600 # action, and can take a long time.
4601 # So we try to avoid locking the file and use 'lock' variable
4602 # inside the config file instead.
4603
4604 my $snapshot_copy_config = sub {
4605     my ($source, $dest) = @_;
4606
4607     foreach my $k (keys %$source) {
4608         next if $k eq 'snapshots';
4609         next if $k eq 'snapstate';
4610         next if $k eq 'snaptime';
4611         next if $k eq 'vmstate';
4612         next if $k eq 'lock';
4613         next if $k eq 'digest';
4614         next if $k eq 'description';
4615         next if $k =~ m/^unused\d+$/;
4616
4617         $dest->{$k} = $source->{$k};
4618     }
4619 };
4620
4621 my $snapshot_apply_config = sub {
4622     my ($conf, $snap) = @_;
4623
4624     # copy snapshot list
4625     my $newconf = {
4626         snapshots => $conf->{snapshots},
4627     };
4628
4629     # keep description and list of unused disks
4630     foreach my $k (keys %$conf) {
4631         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
4632         $newconf->{$k} = $conf->{$k};
4633     }
4634
4635     &$snapshot_copy_config($snap, $newconf);
4636
4637     return $newconf;
4638 };
4639
4640 sub foreach_writable_storage {
4641     my ($conf, $func) = @_;
4642
4643     my $sidhash = {};
4644
4645     foreach my $ds (keys %$conf) {
4646         next if !valid_drivename($ds);
4647
4648         my $drive = parse_drive($ds, $conf->{$ds});
4649         next if !$drive;
4650         next if drive_is_cdrom($drive);
4651
4652         my $volid = $drive->{file};
4653
4654         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4655         $sidhash->{$sid} = $sid if $sid;
4656     }
4657
4658     foreach my $sid (sort keys %$sidhash) {
4659         &$func($sid);
4660     }
4661 }
4662
4663 my $alloc_vmstate_volid = sub {
4664     my ($storecfg, $vmid, $conf, $snapname) = @_;
4665
4666     # Note: we try to be smart when selecting a $target storage
4667
4668     my $target;
4669
4670     # search shared storage first
4671     foreach_writable_storage($conf, sub {
4672         my ($sid) = @_;
4673         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
4674         return if !$scfg->{shared};
4675
4676         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
4677     });
4678
4679     if (!$target) {
4680         # now search local storage
4681         foreach_writable_storage($conf, sub {
4682             my ($sid) = @_;
4683             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
4684             return if $scfg->{shared};
4685
4686             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
4687         });
4688     }
4689
4690     $target = 'local' if !$target;
4691
4692     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
4693     # we abort live save after $conf->{memory}, so we need at max twice that space
4694     my $size = $conf->{memory}*2 + $driver_state_size;
4695
4696     my $name = "vm-$vmid-state-$snapname";
4697     my $scfg = PVE::Storage::storage_config($storecfg, $target);
4698     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
4699     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
4700
4701     return $volid;
4702 };
4703
4704 my $snapshot_prepare = sub {
4705     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
4706
4707     my $snap;
4708
4709     my $updatefn =  sub {
4710
4711         my $conf = load_config($vmid);
4712
4713         die "you can't take a snapshot if it's a template\n"
4714             if is_template($conf);
4715
4716         check_lock($conf);
4717
4718         $conf->{lock} = 'snapshot';
4719
4720         die "snapshot name '$snapname' already used\n"
4721             if defined($conf->{snapshots}->{$snapname});
4722
4723         my $storecfg = PVE::Storage::config();
4724         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
4725
4726         $snap = $conf->{snapshots}->{$snapname} = {};
4727
4728         if ($save_vmstate && check_running($vmid)) {
4729             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
4730         }
4731
4732         &$snapshot_copy_config($conf, $snap);
4733
4734         $snap->{snapstate} = "prepare";
4735         $snap->{snaptime} = time();
4736         $snap->{description} = $comment if $comment;
4737
4738         # always overwrite machine if we save vmstate. This makes sure we
4739         # can restore it later using correct machine type
4740         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
4741
4742         update_config_nolock($vmid, $conf, 1);
4743     };
4744
4745     lock_config($vmid, $updatefn);
4746
4747     return $snap;
4748 };
4749
4750 my $snapshot_commit = sub {
4751     my ($vmid, $snapname) = @_;
4752
4753     my $updatefn = sub {
4754
4755         my $conf = load_config($vmid);
4756
4757         die "missing snapshot lock\n"
4758             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
4759
4760         my $has_machine_config = defined($conf->{machine});
4761
4762         my $snap = $conf->{snapshots}->{$snapname};
4763
4764         die "snapshot '$snapname' does not exist\n" if !defined($snap);
4765
4766         die "wrong snapshot state\n"
4767             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
4768
4769         delete $snap->{snapstate};
4770         delete $conf->{lock};
4771
4772         my $newconf = &$snapshot_apply_config($conf, $snap);
4773
4774         delete $newconf->{machine} if !$has_machine_config;
4775
4776         $newconf->{parent} = $snapname;
4777
4778         update_config_nolock($vmid, $newconf, 1);
4779     };
4780
4781     lock_config($vmid, $updatefn);
4782 };
4783
4784 sub snapshot_rollback {
4785     my ($vmid, $snapname) = @_;
4786
4787     my $snap;
4788
4789     my $prepare = 1;
4790
4791     my $storecfg = PVE::Storage::config();
4792
4793     my $updatefn = sub {
4794
4795         my $conf = load_config($vmid);
4796
4797         die "you can't rollback if vm is a template\n" if is_template($conf);
4798
4799         $snap = $conf->{snapshots}->{$snapname};
4800
4801         die "snapshot '$snapname' does not exist\n" if !defined($snap);
4802
4803         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
4804             if $snap->{snapstate};
4805
4806         if ($prepare) {
4807             check_lock($conf);
4808             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
4809         }
4810
4811         die "unable to rollback vm $vmid: vm is running\n"
4812             if check_running($vmid);
4813
4814         if ($prepare) {
4815             $conf->{lock} = 'rollback';
4816         } else {
4817             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
4818             delete $conf->{lock};
4819         }
4820
4821         my $forcemachine;
4822
4823         if (!$prepare) {
4824             my $has_machine_config = defined($conf->{machine});
4825
4826             # copy snapshot config to current config
4827             $conf = &$snapshot_apply_config($conf, $snap);
4828             $conf->{parent} = $snapname;
4829
4830             # Note: old code did not store 'machine', so we try to be smart
4831             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
4832             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
4833             # we remove the 'machine' configuration if not explicitly specified
4834             # in the original config.
4835             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
4836         }
4837
4838         update_config_nolock($vmid, $conf, 1);
4839
4840         if (!$prepare && $snap->{vmstate}) {
4841             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
4842             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
4843         }
4844     };
4845
4846     lock_config($vmid, $updatefn);
4847
4848     foreach_drive($snap, sub {
4849         my ($ds, $drive) = @_;
4850
4851         return if drive_is_cdrom($drive);
4852
4853         my $volid = $drive->{file};
4854         my $device = "drive-$ds";
4855
4856         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
4857     });
4858
4859     $prepare = 0;
4860     lock_config($vmid, $updatefn);
4861 }
4862
4863 my $savevm_wait = sub {
4864     my ($vmid) = @_;
4865
4866     for(;;) {
4867         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
4868         if (!$stat->{status}) {
4869             die "savevm not active\n";
4870         } elsif ($stat->{status} eq 'active') {
4871             sleep(1);
4872             next;
4873         } elsif ($stat->{status} eq 'completed') {
4874             last;
4875         } else {
4876             die "query-savevm returned status '$stat->{status}'\n";
4877         }
4878     }
4879 };
4880
4881 sub snapshot_create {
4882     my ($vmid, $snapname, $save_vmstate, $freezefs, $comment) = @_;
4883
4884     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
4885
4886     $freezefs = $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
4887
4888     my $drivehash = {};
4889
4890     my $running = check_running($vmid);
4891
4892     eval {
4893         # create internal snapshots of all drives
4894
4895         my $storecfg = PVE::Storage::config();
4896
4897         if ($running) {
4898             if ($snap->{vmstate}) {
4899                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
4900                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
4901                 &$savevm_wait($vmid);
4902             } else {
4903                 vm_mon_cmd($vmid, "savevm-start");
4904             }
4905         };
4906
4907         qga_freezefs($vmid) if $running && $freezefs;
4908
4909         foreach_drive($snap, sub {
4910             my ($ds, $drive) = @_;
4911
4912             return if drive_is_cdrom($drive);
4913
4914             my $volid = $drive->{file};
4915             my $device = "drive-$ds";
4916
4917             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
4918             $drivehash->{$ds} = 1;
4919        });
4920     };
4921     my $err = $@;
4922
4923     eval { qga_unfreezefs($vmid) if $running && $freezefs; };
4924     warn $@ if $@;
4925
4926     eval { vm_mon_cmd($vmid, "savevm-end") if $running; };
4927     warn $@ if $@;
4928
4929     if ($err) {
4930         warn "snapshot create failed: starting cleanup\n";
4931         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
4932         warn $@ if $@;
4933         die $err;
4934     }
4935
4936     &$snapshot_commit($vmid, $snapname);
4937 }
4938
4939 # Note: $drivehash is only set when called from snapshot_create.
4940 sub snapshot_delete {
4941     my ($vmid, $snapname, $force, $drivehash) = @_;
4942
4943     my $prepare = 1;
4944
4945     my $snap;
4946     my $unused = [];
4947
4948     my $unlink_parent = sub {
4949         my ($confref, $new_parent) = @_;
4950
4951         if ($confref->{parent} && $confref->{parent} eq $snapname) {
4952             if ($new_parent) {
4953                 $confref->{parent} = $new_parent;
4954             } else {
4955                 delete $confref->{parent};
4956             }
4957         }
4958     };
4959
4960     my $updatefn =  sub {
4961         my ($remove_drive) = @_;
4962
4963         my $conf = load_config($vmid);
4964
4965         if (!$drivehash) {
4966             check_lock($conf);
4967             die "you can't delete a snapshot if vm is a template\n"
4968                 if is_template($conf);
4969         }
4970
4971         $snap = $conf->{snapshots}->{$snapname};
4972
4973         die "snapshot '$snapname' does not exist\n" if !defined($snap);
4974
4975         # remove parent refs
4976         if (!$prepare) {
4977             &$unlink_parent($conf, $snap->{parent});
4978             foreach my $sn (keys %{$conf->{snapshots}}) {
4979                 next if $sn eq $snapname;
4980                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
4981             }
4982         }
4983
4984         if ($remove_drive) {
4985             if ($remove_drive eq 'vmstate') {
4986                 delete $snap->{$remove_drive};
4987             } else {
4988                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
4989                 my $volid = $drive->{file};
4990                 delete $snap->{$remove_drive};
4991                 add_unused_volume($conf, $volid);
4992             }
4993         }
4994
4995         if ($prepare) {
4996             $snap->{snapstate} = 'delete';
4997         } else {
4998             delete $conf->{snapshots}->{$snapname};
4999             delete $conf->{lock} if $drivehash;
5000             foreach my $volid (@$unused) {
5001                 add_unused_volume($conf, $volid);
5002             }
5003         }
5004
5005         update_config_nolock($vmid, $conf, 1);
5006     };
5007
5008     lock_config($vmid, $updatefn);
5009
5010     # now remove vmstate file
5011
5012     my $storecfg = PVE::Storage::config();
5013
5014     if ($snap->{vmstate}) {
5015         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5016         if (my $err = $@) {
5017             die $err if !$force;
5018             warn $err;
5019         }
5020         # save changes (remove vmstate from snapshot)
5021         lock_config($vmid, $updatefn, 'vmstate') if !$force;
5022     };
5023
5024     # now remove all internal snapshots
5025     foreach_drive($snap, sub {
5026         my ($ds, $drive) = @_;
5027
5028         return if drive_is_cdrom($drive);
5029
5030         my $volid = $drive->{file};
5031         my $device = "drive-$ds";
5032
5033         if (!$drivehash || $drivehash->{$ds}) {
5034             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
5035             if (my $err = $@) {
5036                 die $err if !$force;
5037                 warn $err;
5038             }
5039         }
5040
5041         # save changes (remove drive fron snapshot)
5042         lock_config($vmid, $updatefn, $ds) if !$force;
5043         push @$unused, $volid;
5044     });
5045
5046     # now cleanup config
5047     $prepare = 0;
5048     lock_config($vmid, $updatefn);
5049 }
5050
5051 sub has_feature {
5052     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
5053
5054     my $err;
5055     foreach_drive($conf, sub {
5056         my ($ds, $drive) = @_;
5057
5058         return if drive_is_cdrom($drive);
5059         my $volid = $drive->{file};
5060         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
5061     });
5062
5063     return $err ? 0 : 1;
5064 }
5065
5066 sub template_create {
5067     my ($vmid, $conf, $disk) = @_;
5068
5069     my $storecfg = PVE::Storage::config();
5070
5071     foreach_drive($conf, sub {
5072         my ($ds, $drive) = @_;
5073
5074         return if drive_is_cdrom($drive);
5075         return if $disk && $ds ne $disk;
5076
5077         my $volid = $drive->{file};
5078         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5079
5080         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5081         $drive->{file} = $voliddst;
5082         $conf->{$ds} = print_drive($vmid, $drive);
5083         update_config_nolock($vmid, $conf, 1);
5084     });
5085 }
5086
5087 sub is_template {
5088     my ($conf) = @_;
5089
5090     return 1 if defined $conf->{template} && $conf->{template} == 1;
5091 }
5092
5093 sub qemu_img_convert {
5094     my ($src_volid, $dst_volid, $size, $snapname) = @_;
5095
5096     my $storecfg = PVE::Storage::config();
5097     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5098     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5099
5100     if ($src_storeid && $dst_storeid) {
5101         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5102         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5103
5104         my $src_format = qemu_img_format($src_scfg, $src_volname);
5105         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5106
5107         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5108         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5109
5110         my $cmd = [];
5111         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5112         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5113         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
5114
5115         my $parser = sub {
5116             my $line = shift;
5117             if($line =~ m/\((\S+)\/100\%\)/){
5118                 my $percent = $1;
5119                 my $transferred = int($size * $percent / 100);
5120                 my $remaining = $size - $transferred;
5121
5122                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5123             }
5124
5125         };
5126
5127         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5128         my $err = $@;
5129         die "copy failed: $err" if $err;
5130     }
5131 }
5132
5133 sub qemu_img_format {
5134     my ($scfg, $volname) = @_;
5135
5136     if ($scfg->{path} && $volname =~ m/\.(raw|qcow2|qed|vmdk)$/) {
5137         return $1;
5138     } elsif ($scfg->{type} eq 'iscsi') {
5139         return "host_device";
5140     } else {
5141         return "raw";
5142     }
5143 }
5144
5145 sub qemu_drive_mirror {
5146     my ($vmid, $drive, $dst_volid, $vmiddst, $maxwait) = @_;
5147
5148     my $count = 1;
5149     my $old_len = 0;
5150     my $frozen = undef;
5151
5152     my $storecfg = PVE::Storage::config();
5153     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5154
5155     if ($dst_storeid) {
5156         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5157
5158         my $format;
5159         if ($dst_volname =~ m/\.(raw|qcow2)$/){
5160             $format = $1;
5161         }
5162
5163         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5164
5165         if ($format) {
5166             #fixme : sometime drive-mirror timeout, but works fine after.
5167             # (I have see the problem with big volume > 200GB), so we need to eval
5168             eval { vm_mon_cmd($vmid, "drive-mirror", timeout => 10, device => "drive-$drive", mode => "existing",
5169                               sync => "full", target => $dst_path, format => $format); };
5170         } else {
5171             eval { vm_mon_cmd($vmid, "drive-mirror", timeout => 10, device => "drive-$drive", mode => "existing",
5172                               sync => "full", target => $dst_path); };
5173         }
5174
5175         eval {
5176             while (1) {
5177                 my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5178                 my $stat = @$stats[0];
5179                 die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5180                 die "error job is not mirroring" if $stat->{type} ne "mirror";
5181
5182                 my $transferred = $stat->{offset};
5183                 my $total = $stat->{len};
5184                 my $remaining = $total - $transferred;
5185                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5186
5187                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent %\n";
5188
5189                 last if ($stat->{len} == $stat->{offset});
5190                 if ($old_len == $stat->{offset}) {
5191                     if ($maxwait && $count > $maxwait) {
5192                     # if writes to disk occurs the disk needs to be freezed
5193                     # to be able to complete the migration
5194                         vm_suspend($vmid,1);
5195                         $count = 0;
5196                         $frozen = 1;
5197                     } else {
5198                         $count++ unless $frozen;
5199                     }
5200                 } elsif ($frozen) {
5201                     vm_resume($vmid,1);
5202                     $count = 0;
5203                 }
5204                 $old_len = $stat->{offset};
5205                 sleep 1;
5206             }
5207
5208             if ($vmiddst == $vmid) {
5209                 # switch the disk if source and destination are on the same guest
5210                 vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive");
5211             }
5212         };
5213         if (my $err = $@) {
5214             eval { vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive"); };
5215             die "mirroring error: $err";
5216         }
5217
5218         if ($vmiddst != $vmid) {
5219             # if we clone a disk for a new target vm, we don't switch the disk
5220             vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5221         }
5222     }
5223 }
5224
5225 sub clone_disk {
5226     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5227         $newvmid, $storage, $format, $full, $newvollist) = @_;
5228
5229     my $newvolid;
5230
5231     if (!$full) {
5232         print "create linked clone of drive $drivename ($drive->{file})\n";
5233         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5234         push @$newvollist, $newvolid;
5235     } else {
5236         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5237         $storeid = $storage if $storage;
5238
5239         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5240         if (!$format) {
5241             $format = $drive->{format} || $defFormat;
5242         }
5243
5244         # test if requested format is supported - else use default
5245         my $supported = grep { $_ eq $format } @$validFormats;
5246         $format = $defFormat if !$supported;
5247
5248         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5249
5250         print "create full clone of drive $drivename ($drive->{file})\n";
5251         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5252         push @$newvollist, $newvolid;
5253
5254         if (!$running || $snapname) {
5255             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
5256         } else {
5257             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
5258         }
5259     }
5260
5261     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5262
5263     my $disk = $drive;
5264     $disk->{format} = undef;
5265     $disk->{file} = $newvolid;
5266     $disk->{size} = $size;
5267
5268     return $disk;
5269 }
5270
5271 # this only works if VM is running
5272 sub get_current_qemu_machine {
5273     my ($vmid) = @_;
5274
5275     my $cmd = { execute => 'query-machines', arguments => {} };
5276     my $res = PVE::QemuServer::vm_qmp_command($vmid, $cmd);
5277
5278     my ($current, $default);
5279     foreach my $e (@$res) {
5280         $default = $e->{name} if $e->{'is-default'};
5281         $current = $e->{name} if $e->{'is-current'};
5282     }
5283
5284     # fallback to the default machine if current is not supported by qemu
5285     return $current || $default || 'pc';
5286 }
5287
5288 sub lspci {
5289
5290     my $devices = {};
5291
5292     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
5293             my (undef, $id, $function) = @_;
5294             my $res = { id => $id, function => $function};
5295             push @{$devices->{$id}}, $res;
5296     });
5297
5298     return $devices;
5299 }
5300
5301 1;