PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  35
  36 # Note about locking: we use flock on the config file protect
  37 # against concurent actions.
  38 # Aditionaly, we have a 'lock' setting in the config file. This
  39 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  40 # allowed when such lock is set. But you can ignore this kind of
  41 # lock with the --skiplock flag.
  42
  43 cfs_register_file('/qemu-server/',
  44                   \&parse_vm_config,
  45                   \&write_vm_config);
  46
  47 PVE::JSONSchema::register_standard_option('skiplock', {
  48     description => "Ignore locks - only root is allowed to use this option.",
  49     type => 'boolean',
  50     optional => 1,
  51 });
  52
  53 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  54     description => "Some command save/restore state from this location.",
  55     type => 'string',
  56     maxLength => 128,
  57     optional => 1,
  58 });
  59
  60 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  61     description => "The name of the snapshot.",
  62     type => 'string', format => 'pve-configid',
  63     maxLength => 40,
  64 });
  65
  66 #no warnings 'redefine';
  67
  68 unless(defined(&_VZSYSCALLS_H_)) {
  69     eval 'sub _VZSYSCALLS_H_ () {1;}' unless defined(&_VZSYSCALLS_H_);
  70     require 'sys/syscall.ph';
  71     if(defined(&__x86_64__)) {
  72         eval 'sub __NR_fairsched_vcpus () {499;}' unless defined(&__NR_fairsched_vcpus);
  73         eval 'sub __NR_fairsched_mknod () {504;}' unless defined(&__NR_fairsched_mknod);
  74         eval 'sub __NR_fairsched_rmnod () {505;}' unless defined(&__NR_fairsched_rmnod);
  75         eval 'sub __NR_fairsched_chwt () {506;}' unless defined(&__NR_fairsched_chwt);
  76         eval 'sub __NR_fairsched_mvpr () {507;}' unless defined(&__NR_fairsched_mvpr);
  77         eval 'sub __NR_fairsched_rate () {508;}' unless defined(&__NR_fairsched_rate);
  78         eval 'sub __NR_setluid () {501;}' unless defined(&__NR_setluid);
  79         eval 'sub __NR_setublimit () {502;}' unless defined(&__NR_setublimit);
  80     }
  81     elsif(defined( &__i386__) ) {
  82         eval 'sub __NR_fairsched_mknod () {500;}' unless defined(&__NR_fairsched_mknod);
  83         eval 'sub __NR_fairsched_rmnod () {501;}' unless defined(&__NR_fairsched_rmnod);
  84         eval 'sub __NR_fairsched_chwt () {502;}' unless defined(&__NR_fairsched_chwt);
  85         eval 'sub __NR_fairsched_mvpr () {503;}' unless defined(&__NR_fairsched_mvpr);
  86         eval 'sub __NR_fairsched_rate () {504;}' unless defined(&__NR_fairsched_rate);
  87         eval 'sub __NR_fairsched_vcpus () {505;}' unless defined(&__NR_fairsched_vcpus);
  88         eval 'sub __NR_setluid () {511;}' unless defined(&__NR_setluid);
  89         eval 'sub __NR_setublimit () {512;}' unless defined(&__NR_setublimit);
  90     } else {
  91         die("no fairsched syscall for this arch");
  92     }
  93     require 'asm/ioctl.ph';
  94     eval 'sub KVM_GET_API_VERSION () { &_IO(0xAE, 0x);}' unless defined(&KVM_GET_API_VERSION);
  95 }
  96
  97 sub fairsched_mknod {
  98     my ($parent, $weight, $desired) = @_;
  99
 100     return syscall(&__NR_fairsched_mknod, int($parent), int($weight), int($desired));
 101 }
 102
 103 sub fairsched_rmnod {
 104     my ($id) = @_;
 105
 106     return syscall(&__NR_fairsched_rmnod, int($id));
 107 }
 108
 109 sub fairsched_mvpr {
 110     my ($pid, $newid) = @_;
 111
 112     return syscall(&__NR_fairsched_mvpr, int($pid), int($newid));
 113 }
 114
 115 sub fairsched_vcpus {
 116     my ($id, $vcpus) = @_;
 117
 118     return syscall(&__NR_fairsched_vcpus, int($id), int($vcpus));
 119 }
 120
 121 sub fairsched_rate {
 122     my ($id, $op, $rate) = @_;
 123
 124     return syscall(&__NR_fairsched_rate, int($id), int($op), int($rate));
 125 }
 126
 127 use constant FAIRSCHED_SET_RATE  => 0;
 128 use constant FAIRSCHED_DROP_RATE => 1;
 129 use constant FAIRSCHED_GET_RATE  => 2;
 130
 131 sub fairsched_cpulimit {
 132     my ($id, $limit) = @_;
 133
 134     my $cpulim1024 = int($limit * 1024 / 100);
 135     my $op = $cpulim1024 ? FAIRSCHED_SET_RATE : FAIRSCHED_DROP_RATE;
 136
 137     return fairsched_rate($id, $op, $cpulim1024);
 138 }
 139
 140 my $nodename = PVE::INotify::nodename();
 141
 142 mkdir "/etc/pve/nodes/$nodename";
 143 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 144 mkdir $confdir;
 145
 146 my $var_run_tmpdir = "/var/run/qemu-server";
 147 mkdir $var_run_tmpdir;
 148
 149 my $lock_dir = "/var/lock/qemu-server";
 150 mkdir $lock_dir;
 151
 152 my $pcisysfs = "/sys/bus/pci";
 153
 154 my $confdesc = {
 155     onboot => {
 156         optional => 1,
 157         type => 'boolean',
 158         description => "Specifies whether a VM will be started during system bootup.",
 159         default => 0,
 160     },
 161     autostart => {
 162         optional => 1,
 163         type => 'boolean',
 164         description => "Automatic restart after crash (currently ignored).",
 165         default => 0,
 166     },
 167     hotplug => {
 168         optional => 1,
 169         type => 'boolean',
 170         description => "Allow hotplug for disk and network device",
 171         default => 0,
 172     },
 173     reboot => {
 174         optional => 1,
 175         type => 'boolean',
 176         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 177         default => 1,
 178     },
 179     lock => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Lock/unlock the VM.",
 183         enum => [qw(migrate backup snapshot rollback)],
 184     },
 185     cpulimit => {
 186         optional => 1,
 187         type => 'integer',
 188         description => "Limit of CPU usage in per cent. Note if the computer has 2 CPUs, it has total of 200% CPU time. Value '0' indicates no CPU limit.\n\nNOTE: This option is currently ignored.",
 189         minimum => 0,
 190         default => 0,
 191     },
 192     cpuunits => {
 193         optional => 1,
 194         type => 'integer',
 195         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 196         minimum => 0,
 197         maximum => 500000,
 198         default => 1000,
 199     },
 200     memory => {
 201         optional => 1,
 202         type => 'integer',
 203         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 204         minimum => 16,
 205         default => 512,
 206     },
 207     balloon => {
 208         optional => 1,
 209         type => 'integer',
 210         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 211         minimum => 0,
 212     },
 213     shares => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 217         minimum => 0,
 218         maximum => 50000,
 219         default => 1000,
 220     },
 221     keyboard => {
 222         optional => 1,
 223         type => 'string',
 224         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 225         enum => PVE::Tools::kvmkeymaplist(),
 226         default => 'en-us',
 227     },
 228     name => {
 229         optional => 1,
 230         type => 'string', format => 'dns-name',
 231         description => "Set a name for the VM. Only used on the configuration web interface.",
 232     },
 233     scsihw => {
 234         optional => 1,
 235         type => 'string',
 236         description => "scsi controller model",
 237         enum => [qw(lsi lsi53c810 virtio-scsi-pci megasas pvscsi)],
 238         default => 'lsi',
 239     },
 240     description => {
 241         optional => 1,
 242         type => 'string',
 243         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 244     },
 245     ostype => {
 246         optional => 1,
 247         type => 'string',
 248         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 249         description => <<EODESC,
 250 Used to enable special optimization/features for specific
 251 operating systems:
 252
 253 other  => unspecified OS
 254 wxp    => Microsoft Windows XP
 255 w2k    => Microsoft Windows 2000
 256 w2k3   => Microsoft Windows 2003
 257 w2k8   => Microsoft Windows 2008
 258 wvista => Microsoft Windows Vista
 259 win7   => Microsoft Windows 7
 260 win8   => Microsoft Windows 8/2012
 261 l24    => Linux 2.4 Kernel
 262 l26    => Linux 2.6/3.X Kernel
 263 solaris => solaris/opensolaris/openindiania kernel
 264
 265 other|l24|l26|solaris                       ... no special behaviour
 266 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 267 EODESC
 268     },
 269     boot => {
 270         optional => 1,
 271         type => 'string',
 272         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 273         pattern => '[acdn]{1,4}',
 274         default => 'cdn',
 275     },
 276     bootdisk => {
 277         optional => 1,
 278         type => 'string', format => 'pve-qm-bootdisk',
 279         description => "Enable booting from specified disk.",
 280         pattern => '(ide|sata|scsi|virtio)\d+',
 281     },
 282     smp => {
 283         optional => 1,
 284         type => 'integer',
 285         description => "The number of CPUs. Please use option -sockets instead.",
 286         minimum => 1,
 287         default => 1,
 288     },
 289     sockets => {
 290         optional => 1,
 291         type => 'integer',
 292         description => "The number of CPU sockets.",
 293         minimum => 1,
 294         default => 1,
 295     },
 296     cores => {
 297         optional => 1,
 298         type => 'integer',
 299         description => "The number of cores per socket.",
 300         minimum => 1,
 301         default => 1,
 302     },
 303     maxcpus => {
 304         optional => 1,
 305         type => 'integer',
 306         description => "Maximum cpus for hotplug.",
 307         minimum => 1,
 308         default => 1,
 309     },
 310     acpi => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable ACPI.",
 314         default => 1,
 315     },
 316     agent => {
 317         optional => 1,
 318         type => 'boolean',
 319         description => "Enable/disable Qemu GuestAgent.",
 320         default => 0,
 321     },
 322     kvm => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Enable/disable KVM hardware virtualization.",
 326         default => 1,
 327     },
 328     tdf => {
 329         optional => 1,
 330         type => 'boolean',
 331         description => "Enable/disable time drift fix.",
 332         default => 0,
 333     },
 334     localtime => {
 335         optional => 1,
 336         type => 'boolean',
 337         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 338     },
 339     freeze => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 343     },
 344     vga => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 348         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 349     },
 350     watchdog => {
 351         optional => 1,
 352         type => 'string', format => 'pve-qm-watchdog',
 353         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 354         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 355     },
 356     startdate => {
 357         optional => 1,
 358         type => 'string',
 359         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 360         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 361         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 362         default => 'now',
 363     },
 364     startup => {
 365         optional => 1,
 366         type => 'string', format => 'pve-qm-startup',
 367         typetext => '[[order=]\d+] [,up=\d+] [,down=\d+] ',
 368         description => "Startup and shutdown behavior. Order is a non-negative number defining the general startup order. Shutdown in done with reverse ordering. Additionally you can set the 'up' or 'down' delay in seconds, which specifies a delay to wait before the next VM is started or stopped.",
 369     },
 370     template => {
 371         optional => 1,
 372         type => 'boolean',
 373         description => "Enable/disable Template.",
 374         default => 0,
 375     },
 376     args => {
 377         optional => 1,
 378         type => 'string',
 379         description => <<EODESCR,
 380 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 381
 382 args: -no-reboot -no-hpet
 383 EODESCR
 384     },
 385     tablet => {
 386         optional => 1,
 387         type => 'boolean',
 388         default => 1,
 389         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 390     },
 391     migrate_speed => {
 392         optional => 1,
 393         type => 'integer',
 394         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 395         minimum => 0,
 396         default => 0,
 397     },
 398     migrate_downtime => {
 399         optional => 1,
 400         type => 'number',
 401         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 402         minimum => 0,
 403         default => 0.1,
 404     },
 405     cdrom => {
 406         optional => 1,
 407         type => 'string', format => 'pve-qm-drive',
 408         typetext => 'volume',
 409         description => "This is an alias for option -ide2",
 410     },
 411     cpu => {
 412         optional => 1,
 413         description => "Emulated CPU type.",
 414         type => 'string',
 415         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 416         default => 'kvm64',
 417     },
 418     parent => get_standard_option('pve-snapshot-name', {
 419         optional => 1,
 420         description => "Parent snapshot name. This is used internally, and should not be modified.",
 421     }),
 422     snaptime => {
 423         optional => 1,
 424         description => "Timestamp for snapshots.",
 425         type => 'integer',
 426         minimum => 0,
 427     },
 428     vmstate => {
 429         optional => 1,
 430         type => 'string', format => 'pve-volume-id',
 431         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 432     },
 433     machine => {
 434         description => "Specific the Qemu machine type.",
 435         type => 'string',
 436         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 437         maxLength => 40,
 438         optional => 1,
 439     },
 440     smbios1 => {
 441         description => "Specify SMBIOS type 1 fields.",
 442         type => 'string', format => 'pve-qm-smbios1',
 443         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 444         maxLength => 256,
 445         optional => 1,
 446     },
 447 };
 448
 449 # what about other qemu settings ?
 450 #cpu => 'string',
 451 #machine => 'string',
 452 #fda => 'file',
 453 #fdb => 'file',
 454 #mtdblock => 'file',
 455 #sd => 'file',
 456 #pflash => 'file',
 457 #snapshot => 'bool',
 458 #bootp => 'file',
 459 ##tftp => 'dir',
 460 ##smb => 'dir',
 461 #kernel => 'file',
 462 #append => 'string',
 463 #initrd => 'file',
 464 ##soundhw => 'string',
 465
 466 while (my ($k, $v) = each %$confdesc) {
 467     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 468 }
 469
 470 my $MAX_IDE_DISKS = 4;
 471 my $MAX_SCSI_DISKS = 14;
 472 my $MAX_VIRTIO_DISKS = 16;
 473 my $MAX_SATA_DISKS = 6;
 474 my $MAX_USB_DEVICES = 5;
 475 my $MAX_NETS = 32;
 476 my $MAX_UNUSED_DISKS = 8;
 477 my $MAX_HOSTPCI_DEVICES = 4;
 478 my $MAX_SERIAL_PORTS = 4;
 479 my $MAX_PARALLEL_PORTS = 3;
 480
 481 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 482                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3'];
 483 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 484
 485 my $netdesc = {
 486     optional => 1,
 487     type => 'string', format => 'pve-qm-net',
 488     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>][,tag=<vlanid>][,firewall=0|1]",
 489     description => <<EODESCR,
 490 Specify network devices.
 491
 492 MODEL is one of: $nic_model_list_txt
 493
 494 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 495 automatically generated if not specified.
 496
 497 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 498
 499 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 500
 501 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 502
 503 10.0.2.2   Gateway
 504 10.0.2.3   DNS Server
 505 10.0.2.4   SMB Server
 506
 507 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 508
 509 EODESCR
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 512
 513 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 514     $confdesc->{"net$i"} = $netdesc;
 515 }
 516
 517 my $drivename_hash;
 518
 519 my $idedesc = {
 520     optional => 1,
 521     type => 'string', format => 'pve-qm-drive',
 522     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 523     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 524 };
 525 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 526
 527 my $scsidesc = {
 528     optional => 1,
 529     type => 'string', format => 'pve-qm-drive',
 530     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 531     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 532 };
 533 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 534
 535 my $satadesc = {
 536     optional => 1,
 537     type => 'string', format => 'pve-qm-drive',
 538     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 539     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 540 };
 541 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 542
 543 my $virtiodesc = {
 544     optional => 1,
 545     type => 'string', format => 'pve-qm-drive',
 546     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 547     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 548 };
 549 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 550
 551 my $usbdesc = {
 552     optional => 1,
 553     type => 'string', format => 'pve-qm-usb-device',
 554     typetext => 'host=HOSTUSBDEVICE|spice',
 555     description => <<EODESCR,
 556 Configure an USB device (n is 0 to 4). This can be used to
 557 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 558
 559 'bus-port(.port)*' (decimal numbers) or
 560 'vendor_id:product_id' (hexadeciaml numbers)
 561
 562 You can use the 'lsusb -t' command to list existing usb devices.
 563
 564 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 565
 566 The value 'spice' can be used to add a usb redirection devices for spice.
 567
 568 EODESCR
 569 };
 570 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 571
 572 my $hostpcidesc = {
 573         optional => 1,
 574         type => 'string', format => 'pve-qm-hostpci',
 575         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 576         description => <<EODESCR,
 577 Map host pci devices. HOSTPCIDEVICE syntax is:
 578
 579 'bus:dev.func' (hexadecimal numbers)
 580
 581 You can us the 'lspci' command to list existing pci devices.
 582
 583 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 584
 585 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 586
 587 Experimental: user reported problems with this option.
 588 EODESCR
 589 };
 590 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 591
 592 my $serialdesc = {
 593         optional => 1,
 594         type => 'string',
 595         pattern => '(/dev/ttyS\d+|socket)',
 596         description =>  <<EODESCR,
 597 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device, or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 598
 599 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 600
 601 Experimental: user reported problems with this option.
 602 EODESCR
 603 };
 604
 605 my $paralleldesc= {
 606         optional => 1,
 607         type => 'string',
 608         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 609         description =>  <<EODESCR,
 610 Map host parallel devices (n is 0 to 2).
 611
 612 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 613
 614 Experimental: user reported problems with this option.
 615 EODESCR
 616 };
 617
 618 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 619     $confdesc->{"parallel$i"} = $paralleldesc;
 620 }
 621
 622 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 623     $confdesc->{"serial$i"} = $serialdesc;
 624 }
 625
 626 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 627     $confdesc->{"hostpci$i"} = $hostpcidesc;
 628 }
 629
 630 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 631     $drivename_hash->{"ide$i"} = 1;
 632     $confdesc->{"ide$i"} = $idedesc;
 633 }
 634
 635 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 636     $drivename_hash->{"sata$i"} = 1;
 637     $confdesc->{"sata$i"} = $satadesc;
 638 }
 639
 640 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 641     $drivename_hash->{"scsi$i"} = 1;
 642     $confdesc->{"scsi$i"} = $scsidesc ;
 643 }
 644
 645 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 646     $drivename_hash->{"virtio$i"} = 1;
 647     $confdesc->{"virtio$i"} = $virtiodesc;
 648 }
 649
 650 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 651     $confdesc->{"usb$i"} = $usbdesc;
 652 }
 653
 654 my $unuseddesc = {
 655     optional => 1,
 656     type => 'string', format => 'pve-volume-id',
 657     description => "Reference to unused volumes.",
 658 };
 659
 660 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 661     $confdesc->{"unused$i"} = $unuseddesc;
 662 }
 663
 664 my $kvm_api_version = 0;
 665
 666 sub kvm_version {
 667
 668     return $kvm_api_version if $kvm_api_version;
 669
 670     my $fh = IO::File->new("</dev/kvm") ||
 671         return 0;
 672
 673     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 674         $kvm_api_version = $v;
 675     }
 676
 677     $fh->close();
 678
 679     return  $kvm_api_version;
 680 }
 681
 682 my $kvm_user_version;
 683
 684 sub kvm_user_version {
 685
 686     return $kvm_user_version if $kvm_user_version;
 687
 688     $kvm_user_version = 'unknown';
 689
 690     my $tmp = `kvm -help 2>/dev/null`;
 691
 692     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 693         $kvm_user_version = $2;
 694     }
 695
 696     return $kvm_user_version;
 697
 698 }
 699
 700 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 701
 702 sub disknames {
 703     # order is important - used to autoselect boot disk
 704     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 705             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 706             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 707             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 708 }
 709
 710 sub valid_drivename {
 711     my $dev = shift;
 712
 713     return defined($drivename_hash->{$dev});
 714 }
 715
 716 sub option_exists {
 717     my $key = shift;
 718     return defined($confdesc->{$key});
 719 }
 720
 721 sub nic_models {
 722     return $nic_model_list;
 723 }
 724
 725 sub os_list_description {
 726
 727     return {
 728         other => 'Other',
 729         wxp => 'Windows XP',
 730         w2k => 'Windows 2000',
 731         w2k3 =>, 'Windows 2003',
 732         w2k8 => 'Windows 2008',
 733         wvista => 'Windows Vista',
 734         win7 => 'Windows 7',
 735         win8 => 'Windows 8/2012',
 736         l24 => 'Linux 2.4',
 737         l26 => 'Linux 2.6',
 738     };
 739 }
 740
 741 my $cdrom_path;
 742
 743 sub get_cdrom_path {
 744
 745     return  $cdrom_path if $cdrom_path;
 746
 747     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 748     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 749     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 750 }
 751
 752 sub get_iso_path {
 753     my ($storecfg, $vmid, $cdrom) = @_;
 754
 755     if ($cdrom eq 'cdrom') {
 756         return get_cdrom_path();
 757     } elsif ($cdrom eq 'none') {
 758         return '';
 759     } elsif ($cdrom =~ m|^/|) {
 760         return $cdrom;
 761     } else {
 762         return PVE::Storage::path($storecfg, $cdrom);
 763     }
 764 }
 765
 766 # try to convert old style file names to volume IDs
 767 sub filename_to_volume_id {
 768     my ($vmid, $file, $media) = @_;
 769
 770     if (!($file eq 'none' || $file eq 'cdrom' ||
 771           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 772
 773         return undef if $file =~ m|/|;
 774
 775         if ($media && $media eq 'cdrom') {
 776             $file = "local:iso/$file";
 777         } else {
 778             $file = "local:$vmid/$file";
 779         }
 780     }
 781
 782     return $file;
 783 }
 784
 785 sub verify_media_type {
 786     my ($opt, $vtype, $media) = @_;
 787
 788     return if !$media;
 789
 790     my $etype;
 791     if ($media eq 'disk') {
 792         $etype = 'images';
 793     } elsif ($media eq 'cdrom') {
 794         $etype = 'iso';
 795     } else {
 796         die "internal error";
 797     }
 798
 799     return if ($vtype eq $etype);
 800
 801     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 802 }
 803
 804 sub cleanup_drive_path {
 805     my ($opt, $storecfg, $drive) = @_;
 806
 807     # try to convert filesystem paths to volume IDs
 808
 809     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 810         ($drive->{file} !~ m|^/dev/.+|) &&
 811         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 812         ($drive->{file} !~ m/^\d+$/)) {
 813         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 814         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 815         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 816         verify_media_type($opt, $vtype, $drive->{media});
 817         $drive->{file} = $volid;
 818     }
 819
 820     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 821 }
 822
 823 sub create_conf_nolock {
 824     my ($vmid, $settings) = @_;
 825
 826     my $filename = config_file($vmid);
 827
 828     die "configuration file '$filename' already exists\n" if -f $filename;
 829
 830     my $defaults = load_defaults();
 831
 832     $settings->{name} = "vm$vmid" if !$settings->{name};
 833     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 834
 835     my $data = '';
 836     foreach my $opt (keys %$settings) {
 837         next if !$confdesc->{$opt};
 838
 839         my $value = $settings->{$opt};
 840         next if !$value;
 841
 842         $data .= "$opt: $value\n";
 843     }
 844
 845     PVE::Tools::file_set_contents($filename, $data);
 846 }
 847
 848 my $parse_size = sub {
 849     my ($value) = @_;
 850
 851     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 852     my ($size, $unit) = ($1, $3);
 853     if ($unit) {
 854         if ($unit eq 'K') {
 855             $size = $size * 1024;
 856         } elsif ($unit eq 'M') {
 857             $size = $size * 1024 * 1024;
 858         } elsif ($unit eq 'G') {
 859             $size = $size * 1024 * 1024 * 1024;
 860         }
 861     }
 862     return int($size);
 863 };
 864
 865 my $format_size = sub {
 866     my ($size) = @_;
 867
 868     $size = int($size);
 869
 870     my $kb = int($size/1024);
 871     return $size if $kb*1024 != $size;
 872
 873     my $mb = int($kb/1024);
 874     return "${kb}K" if $mb*1024 != $kb;
 875
 876     my $gb = int($mb/1024);
 877     return "${mb}M" if $gb*1024 != $mb;
 878
 879     return "${gb}G";
 880 };
 881
 882 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 883 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 884 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 885 #        [,aio=native|threads][,discard=ignore|on]
 886
 887 sub parse_drive {
 888     my ($key, $data) = @_;
 889
 890     my $res = {};
 891
 892     # $key may be undefined - used to verify JSON parameters
 893     if (!defined($key)) {
 894         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 895         $res->{index} = 0;
 896     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 897         $res->{interface} = $1;
 898         $res->{index} = $2;
 899     } else {
 900         return undef;
 901     }
 902
 903     foreach my $p (split (/,/, $data)) {
 904         next if $p =~ m/^\s*$/;
 905
 906         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard)=(.+)$/) {
 907             my ($k, $v) = ($1, $2);
 908
 909             $k = 'file' if $k eq 'volume';
 910
 911             return undef if defined $res->{$k};
 912
 913             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 914                 return undef if !$v || $v !~ m/^\d+/;
 915                 $k = "m$k";
 916                 $v = sprintf("%.3f", $v / (1024*1024));
 917             }
 918             $res->{$k} = $v;
 919         } else {
 920             if (!$res->{file} && $p !~ m/=/) {
 921                 $res->{file} = $p;
 922             } else {
 923                 return undef;
 924             }
 925         }
 926     }
 927
 928     return undef if !$res->{file};
 929
 930     if($res->{file} =~ m/\.(raw|cow|qcow|qcow2|vmdk|cloop)$/){
 931         $res->{format} = $1;
 932     }
 933
 934     return undef if $res->{cache} &&
 935         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 936     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 937     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 938     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 939     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 940     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 941     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 942     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qcow2|vmdk|cloop)$/;
 943     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 944     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 945     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 946     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 947     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 948
 949     return undef if $res->{mbps_rd} && $res->{mbps};
 950     return undef if $res->{mbps_wr} && $res->{mbps};
 951
 952     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 953     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 954     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 955     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 956     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 957     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 958
 959     return undef if $res->{iops_rd} && $res->{iops};
 960     return undef if $res->{iops_wr} && $res->{iops};
 961
 962
 963     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 964     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 965     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 966     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 967     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 968     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 969
 970
 971     if ($res->{size}) {
 972         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 973     }
 974
 975     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 976         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 977         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 978         return undef if $res->{interface} eq 'virtio';
 979     }
 980
 981     # rerror does not work with scsi drives
 982     if ($res->{rerror}) {
 983         return undef if $res->{interface} eq 'scsi';
 984     }
 985
 986     return $res;
 987 }
 988
 989 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
 990
 991 sub print_drive {
 992     my ($vmid, $drive) = @_;
 993
 994     my $opts = '';
 995     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup') {
 996         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 997     }
 998
 999     if ($drive->{size}) {
1000         $opts .= ",size=" . &$format_size($drive->{size});
1001     }
1002
1003     return "$drive->{file}$opts";
1004 }
1005
1006 sub scsi_inquiry {
1007     my($fh, $noerr) = @_;
1008
1009     my $SG_IO = 0x2285;
1010     my $SG_GET_VERSION_NUM = 0x2282;
1011
1012     my $versionbuf = "\x00" x 8;
1013     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1014     if (!$ret) {
1015         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1016         return undef;
1017     }
1018     my $version = unpack("I", $versionbuf);
1019     if ($version < 30000) {
1020         die "scsi generic interface too old\n"  if !$noerr;
1021         return undef;
1022     }
1023
1024     my $buf = "\x00" x 36;
1025     my $sensebuf = "\x00" x 8;
1026     my $cmd = pack("C x3 C x1", 0x12, 36);
1027
1028     # see /usr/include/scsi/sg.h
1029     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1030
1031     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1032                       length($sensebuf), 0, length($buf), $buf,
1033                       $cmd, $sensebuf, 6000);
1034
1035     $ret = ioctl($fh, $SG_IO, $packet);
1036     if (!$ret) {
1037         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1038         return undef;
1039     }
1040
1041     my @res = unpack($sg_io_hdr_t, $packet);
1042     if ($res[17] || $res[18]) {
1043         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1044         return undef;
1045     }
1046
1047     my $res = {};
1048     (my $byte0, my $byte1, $res->{vendor},
1049      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1050
1051     $res->{removable} = $byte1 & 128 ? 1 : 0;
1052     $res->{type} = $byte0 & 31;
1053
1054     return $res;
1055 }
1056
1057 sub path_is_scsi {
1058     my ($path) = @_;
1059
1060     my $fh = IO::File->new("+<$path") || return undef;
1061     my $res = scsi_inquiry($fh, 1);
1062     close($fh);
1063
1064     return $res;
1065 }
1066
1067 sub machine_type_is_q35 {
1068     my ($conf) = @_;
1069
1070     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1071 }
1072
1073 sub print_tabletdevice_full {
1074     my ($conf) = @_;
1075
1076     my $q35 = machine_type_is_q35($conf);
1077
1078     # we use uhci for old VMs because tablet driver was buggy in older qemu
1079     my $usbbus = $q35 ? "ehci" : "uhci";
1080
1081     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1082 }
1083
1084 sub print_drivedevice_full {
1085     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1086
1087     my $device = '';
1088     my $maxdev = 0;
1089
1090     if ($drive->{interface} eq 'virtio') {
1091         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1092         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1093     } elsif ($drive->{interface} eq 'scsi') {
1094         $maxdev = ($conf->{scsihw} && ($conf->{scsihw} !~ m/^lsi/)) ? 256 : 7;
1095         my $controller = int($drive->{index} / $maxdev);
1096         my $unit = $drive->{index} % $maxdev;
1097         my $devicetype = 'hd';
1098         my $path = '';
1099         if (drive_is_cdrom($drive)) {
1100               $devicetype = 'cd';
1101         } else {
1102               if ($drive->{file} =~ m|^/|) {
1103                   $path = $drive->{file};
1104               } else {
1105                   $path = PVE::Storage::path($storecfg, $drive->{file});
1106               }
1107
1108               if($path =~ m/^iscsi\:\/\//){
1109                   $devicetype = 'generic';
1110               } else {
1111                   if (my $info = path_is_scsi($path)) {
1112                       if ($info->{type} == 0) {
1113                           $devicetype = 'block';
1114                       } elsif ($info->{type} == 1) { # tape
1115                           $devicetype = 'generic';
1116                       }
1117                   }
1118               }
1119          }
1120
1121         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1122             $device = "scsi-$devicetype,bus=scsihw$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1123         } else {
1124             $device = "scsi-$devicetype,bus=scsihw$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1125         }
1126
1127     } elsif ($drive->{interface} eq 'ide'){
1128         $maxdev = 2;
1129         my $controller = int($drive->{index} / $maxdev);
1130         my $unit = $drive->{index} % $maxdev;
1131         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1132
1133         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1134     } elsif ($drive->{interface} eq 'sata'){
1135         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1136         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1137         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1138     } elsif ($drive->{interface} eq 'usb') {
1139         die "implement me";
1140         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1141     } else {
1142         die "unsupported interface type";
1143     }
1144
1145     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1146
1147     return $device;
1148 }
1149
1150 sub get_initiator_name {
1151     my $initiator;
1152
1153     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1154     while (defined(my $line = <$fh>)) {
1155         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1156         $initiator = $1;
1157         last;
1158     }
1159     $fh->close();
1160
1161     return $initiator;
1162 }
1163
1164 sub print_drive_full {
1165     my ($storecfg, $vmid, $drive) = @_;
1166
1167     my $opts = '';
1168     foreach my $o (@qemu_drive_options) {
1169         next if $o eq 'bootindex';
1170         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1171     }
1172
1173     foreach my $o (qw(bps bps_rd bps_wr)) {
1174         my $v = $drive->{"m$o"};
1175         $opts .= ",$o=" . int($v*1024*1024) if $v;
1176     }
1177
1178     # use linux-aio by default (qemu default is threads)
1179     $opts .= ",aio=native" if !$drive->{aio};
1180
1181     my $path;
1182     my $volid = $drive->{file};
1183     if (drive_is_cdrom($drive)) {
1184         $path = get_iso_path($storecfg, $vmid, $volid);
1185     } else {
1186         if ($volid =~ m|^/|) {
1187             $path = $volid;
1188         } else {
1189             $path = PVE::Storage::path($storecfg, $volid);
1190         }
1191     }
1192
1193     $opts .= ",cache=none" if !$drive->{cache} && !drive_is_cdrom($drive);
1194
1195     my $pathinfo = $path ? "file=$path," : '';
1196
1197     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1198 }
1199
1200 sub print_netdevice_full {
1201     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1202
1203     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1204
1205     my $device = $net->{model};
1206     if ($net->{model} eq 'virtio') {
1207          $device = 'virtio-net-pci';
1208      };
1209
1210     # qemu > 0.15 always try to boot from network - we disable that by
1211     # not loading the pxe rom file
1212     my $extra = ($bootorder !~ m/n/) ? "romfile=," : '';
1213     my $pciaddr = print_pci_addr("$netid", $bridges);
1214     my $tmpstr = "$device,${extra}mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1215     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1216         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1217         my $vectors = $net->{queues} * 2 + 2;
1218         $tmpstr .= ",vectors=$vectors,mq=on";
1219     }
1220     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1221     return $tmpstr;
1222 }
1223
1224 sub print_netdev_full {
1225     my ($vmid, $conf, $net, $netid) = @_;
1226
1227     my $i = '';
1228     if ($netid =~ m/^net(\d+)$/) {
1229         $i = int($1);
1230     }
1231
1232     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1233
1234     my $ifname = "tap${vmid}i$i";
1235
1236     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1237     die "interface name '$ifname' is too long (max 15 character)\n"
1238         if length($ifname) >= 16;
1239
1240     my $vhostparam = '';
1241     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1242
1243     my $vmname = $conf->{name} || "vm$vmid";
1244
1245     my $netdev = "";
1246
1247     if ($net->{bridge}) {
1248         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1249     } else {
1250         $netdev = "type=user,id=$netid,hostname=$vmname";
1251     }
1252
1253     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1254
1255     return $netdev;
1256 }
1257
1258 sub drive_is_cdrom {
1259     my ($drive) = @_;
1260
1261     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1262
1263 }
1264
1265 sub parse_hostpci {
1266     my ($value) = @_;
1267
1268     return undef if !$value;
1269
1270
1271     my @list = split(/,/, $value);
1272     my $found;
1273
1274     my $res = {};
1275     foreach my $kv (@list) {
1276
1277         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1278             $found = 1;
1279             if(defined($4)){
1280                 push @{$res->{pciid}}, { id => $2 , function => $4};
1281
1282             }else{
1283                 my $pcidevices = lspci($2);
1284                 $res->{pciid} = $pcidevices->{$2};
1285             }
1286         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1287             $res->{driver} = $1;
1288         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1289             $res->{rombar} = $1;
1290         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1291             $res->{'x-vga'} = $1;
1292         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1293             $res->{pcie} = 1 if $1 == 1;
1294         } else {
1295             warn "unknown hostpci setting '$kv'\n";
1296         }
1297     }
1298
1299     return undef if !$found;
1300
1301     return $res;
1302 }
1303
1304 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1305 sub parse_net {
1306     my ($data) = @_;
1307
1308     my $res = {};
1309
1310     foreach my $kvp (split(/,/, $data)) {
1311
1312         if ($kvp =~ m/^(ne2k_pci|e1000|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1313             my $model = lc($1);
1314             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1315             $res->{model} = $model;
1316             $res->{macaddr} = $mac;
1317         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1318             $res->{bridge} = $1;
1319         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1320             $res->{queues} = $1;
1321         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1322             $res->{rate} = $1;
1323         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1324             $res->{tag} = $1;
1325         } elsif ($kvp =~ m/^firewall=(\d+)$/) {
1326             $res->{firewall} = $1;
1327         } else {
1328             return undef;
1329         }
1330
1331     }
1332
1333     return undef if !$res->{model};
1334
1335     return $res;
1336 }
1337
1338 sub print_net {
1339     my $net = shift;
1340
1341     my $res = "$net->{model}";
1342     $res .= "=$net->{macaddr}" if $net->{macaddr};
1343     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1344     $res .= ",rate=$net->{rate}" if $net->{rate};
1345     $res .= ",tag=$net->{tag}" if $net->{tag};
1346     $res .= ",firewall=$net->{firewall}" if $net->{firewall};
1347
1348     return $res;
1349 }
1350
1351 sub add_random_macs {
1352     my ($settings) = @_;
1353
1354     foreach my $opt (keys %$settings) {
1355         next if $opt !~ m/^net(\d+)$/;
1356         my $net = parse_net($settings->{$opt});
1357         next if !$net;
1358         $settings->{$opt} = print_net($net);
1359     }
1360 }
1361
1362 sub add_unused_volume {
1363     my ($config, $volid) = @_;
1364
1365     my $key;
1366     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1367         my $test = "unused$ind";
1368         if (my $vid = $config->{$test}) {
1369             return if $vid eq $volid; # do not add duplicates
1370         } else {
1371             $key = $test;
1372         }
1373     }
1374
1375     die "To many unused volume - please delete them first.\n" if !$key;
1376
1377     $config->{$key} = $volid;
1378
1379     return $key;
1380 }
1381
1382 my $valid_smbios1_options = {
1383     manufacturer => '\S+',
1384     product => '\S+',
1385     version => '\S+',
1386     serial => '\S+',
1387     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1388     sku => '\S+',
1389     family => '\S+',
1390 };
1391
1392 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1393 sub parse_smbios1 {
1394     my ($data) = @_;
1395
1396     my $res = {};
1397
1398     foreach my $kvp (split(/,/, $data)) {
1399         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1400         my ($k, $v) = split(/=/, $kvp);
1401         return undef if !defined($k) || !defined($v);
1402         return undef if !$valid_smbios1_options->{$k};
1403         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1404         $res->{$k} = $v;
1405     }
1406
1407     return $res;
1408 }
1409
1410 sub print_smbios1 {
1411     my ($smbios1) = @_;
1412
1413     my $data = '';
1414     foreach my $k (keys %$smbios1) {
1415         next if !defined($smbios1->{$k});
1416         next if !$valid_smbios1_options->{$k};
1417         $data .= ',' if $data;
1418         $data .= "$k=$smbios1->{$k}";
1419     }
1420     return $data;
1421 }
1422
1423 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1424 sub verify_smbios1 {
1425     my ($value, $noerr) = @_;
1426
1427     return $value if parse_smbios1($value);
1428
1429     return undef if $noerr;
1430
1431     die "unable to parse smbios (type 1) options\n";
1432 }
1433
1434 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1435 sub verify_bootdisk {
1436     my ($value, $noerr) = @_;
1437
1438     return $value if valid_drivename($value);
1439
1440     return undef if $noerr;
1441
1442     die "invalid boot disk '$value'\n";
1443 }
1444
1445 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1446 sub verify_net {
1447     my ($value, $noerr) = @_;
1448
1449     return $value if parse_net($value);
1450
1451     return undef if $noerr;
1452
1453     die "unable to parse network options\n";
1454 }
1455
1456 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1457 sub verify_drive {
1458     my ($value, $noerr) = @_;
1459
1460     return $value if parse_drive(undef, $value);
1461
1462     return undef if $noerr;
1463
1464     die "unable to parse drive options\n";
1465 }
1466
1467 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1468 sub verify_hostpci {
1469     my ($value, $noerr) = @_;
1470
1471     return $value if parse_hostpci($value);
1472
1473     return undef if $noerr;
1474
1475     die "unable to parse pci id\n";
1476 }
1477
1478 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1479 sub verify_watchdog {
1480     my ($value, $noerr) = @_;
1481
1482     return $value if parse_watchdog($value);
1483
1484     return undef if $noerr;
1485
1486     die "unable to parse watchdog options\n";
1487 }
1488
1489 sub parse_watchdog {
1490     my ($value) = @_;
1491
1492     return undef if !$value;
1493
1494     my $res = {};
1495
1496     foreach my $p (split(/,/, $value)) {
1497         next if $p =~ m/^\s*$/;
1498
1499         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1500             $res->{model} = $2;
1501         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1502             $res->{action} = $2;
1503         } else {
1504             return undef;
1505         }
1506     }
1507
1508     return $res;
1509 }
1510
1511 PVE::JSONSchema::register_format('pve-qm-startup', \&verify_startup);
1512 sub verify_startup {
1513     my ($value, $noerr) = @_;
1514
1515     return $value if parse_startup($value);
1516
1517     return undef if $noerr;
1518
1519     die "unable to parse startup options\n";
1520 }
1521
1522 sub parse_startup {
1523     my ($value) = @_;
1524
1525     return undef if !$value;
1526
1527     my $res = {};
1528
1529     foreach my $p (split(/,/, $value)) {
1530         next if $p =~ m/^\s*$/;
1531
1532         if ($p =~ m/^(order=)?(\d+)$/) {
1533             $res->{order} = $2;
1534         } elsif ($p =~ m/^up=(\d+)$/) {
1535             $res->{up} = $1;
1536         } elsif ($p =~ m/^down=(\d+)$/) {
1537             $res->{down} = $1;
1538         } else {
1539             return undef;
1540         }
1541     }
1542
1543     return $res;
1544 }
1545
1546 sub parse_usb_device {
1547     my ($value) = @_;
1548
1549     return undef if !$value;
1550
1551     my @dl = split(/,/, $value);
1552     my $found;
1553
1554     my $res = {};
1555     foreach my $v (@dl) {
1556         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1557             $found = 1;
1558             $res->{vendorid} = $2;
1559             $res->{productid} = $4;
1560         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1561             $found = 1;
1562             $res->{hostbus} = $1;
1563             $res->{hostport} = $2;
1564         } elsif ($v =~ m/^spice$/) {
1565             $found = 1;
1566             $res->{spice} = 1;
1567         } else {
1568             return undef;
1569         }
1570     }
1571     return undef if !$found;
1572
1573     return $res;
1574 }
1575
1576 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1577 sub verify_usb_device {
1578     my ($value, $noerr) = @_;
1579
1580     return $value if parse_usb_device($value);
1581
1582     return undef if $noerr;
1583
1584     die "unable to parse usb device\n";
1585 }
1586
1587 # add JSON properties for create and set function
1588 sub json_config_properties {
1589     my $prop = shift;
1590
1591     foreach my $opt (keys %$confdesc) {
1592         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1593         $prop->{$opt} = $confdesc->{$opt};
1594     }
1595
1596     return $prop;
1597 }
1598
1599 sub check_type {
1600     my ($key, $value) = @_;
1601
1602     die "unknown setting '$key'\n" if !$confdesc->{$key};
1603
1604     my $type = $confdesc->{$key}->{type};
1605
1606     if (!defined($value)) {
1607         die "got undefined value\n";
1608     }
1609
1610     if ($value =~ m/[\n\r]/) {
1611         die "property contains a line feed\n";
1612     }
1613
1614     if ($type eq 'boolean') {
1615         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1616         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1617         die "type check ('boolean') failed - got '$value'\n";
1618     } elsif ($type eq 'integer') {
1619         return int($1) if $value =~ m/^(\d+)$/;
1620         die "type check ('integer') failed - got '$value'\n";
1621     } elsif ($type eq 'number') {
1622         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1623         die "type check ('number') failed - got '$value'\n";
1624     } elsif ($type eq 'string') {
1625         if (my $fmt = $confdesc->{$key}->{format}) {
1626             if ($fmt eq 'pve-qm-drive') {
1627                 # special case - we need to pass $key to parse_drive()
1628                 my $drive = parse_drive($key, $value);
1629                 return $value if $drive;
1630                 die "unable to parse drive options\n";
1631             }
1632             PVE::JSONSchema::check_format($fmt, $value);
1633             return $value;
1634         }
1635         $value =~ s/^\"(.*)\"$/$1/;
1636         return $value;
1637     } else {
1638         die "internal error"
1639     }
1640 }
1641
1642 sub lock_config_full {
1643     my ($vmid, $timeout, $code, @param) = @_;
1644
1645     my $filename = config_file_lock($vmid);
1646
1647     my $res = lock_file($filename, $timeout, $code, @param);
1648
1649     die $@ if $@;
1650
1651     return $res;
1652 }
1653
1654 sub lock_config_mode {
1655     my ($vmid, $timeout, $shared, $code, @param) = @_;
1656
1657     my $filename = config_file_lock($vmid);
1658
1659     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1660
1661     die $@ if $@;
1662
1663     return $res;
1664 }
1665
1666 sub lock_config {
1667     my ($vmid, $code, @param) = @_;
1668
1669     return lock_config_full($vmid, 10, $code, @param);
1670 }
1671
1672 sub cfs_config_path {
1673     my ($vmid, $node) = @_;
1674
1675     $node = $nodename if !$node;
1676     return "nodes/$node/qemu-server/$vmid.conf";
1677 }
1678
1679 sub check_iommu_support{
1680     #fixme : need to check IOMMU support
1681     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1682
1683     my $iommu=1;
1684     return $iommu;
1685
1686 }
1687
1688 sub config_file {
1689     my ($vmid, $node) = @_;
1690
1691     my $cfspath = cfs_config_path($vmid, $node);
1692     return "/etc/pve/$cfspath";
1693 }
1694
1695 sub config_file_lock {
1696     my ($vmid) = @_;
1697
1698     return "$lock_dir/lock-$vmid.conf";
1699 }
1700
1701 sub touch_config {
1702     my ($vmid) = @_;
1703
1704     my $conf = config_file($vmid);
1705     utime undef, undef, $conf;
1706 }
1707
1708 sub destroy_vm {
1709     my ($storecfg, $vmid, $keep_empty_config) = @_;
1710
1711     my $conffile = config_file($vmid);
1712
1713     my $conf = load_config($vmid);
1714
1715     check_lock($conf);
1716
1717     # only remove disks owned by this VM
1718     foreach_drive($conf, sub {
1719         my ($ds, $drive) = @_;
1720
1721         return if drive_is_cdrom($drive);
1722
1723         my $volid = $drive->{file};
1724
1725         return if !$volid || $volid =~ m|^/|;
1726
1727         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1728         return if !$path || !$owner || ($owner != $vmid);
1729
1730         PVE::Storage::vdisk_free($storecfg, $volid);
1731     });
1732
1733     if ($keep_empty_config) {
1734         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1735     } else {
1736         unlink $conffile;
1737     }
1738
1739     # also remove unused disk
1740     eval {
1741         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1742
1743         eval {
1744             PVE::Storage::foreach_volid($dl, sub {
1745                 my ($volid, $sid, $volname, $d) = @_;
1746                 PVE::Storage::vdisk_free($storecfg, $volid);
1747             });
1748         };
1749         warn $@ if $@;
1750
1751     };
1752     warn $@ if $@;
1753 }
1754
1755 sub load_config {
1756     my ($vmid, $node) = @_;
1757
1758     my $cfspath = cfs_config_path($vmid, $node);
1759
1760     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1761
1762     die "no such VM ('$vmid')\n" if !defined($conf);
1763
1764     return $conf;
1765 }
1766
1767 sub parse_vm_config {
1768     my ($filename, $raw) = @_;
1769
1770     return undef if !defined($raw);
1771
1772     my $res = {
1773         digest => Digest::SHA::sha1_hex($raw),
1774         snapshots => {},
1775     };
1776
1777     $filename =~ m|/qemu-server/(\d+)\.conf$|
1778         || die "got strange filename '$filename'";
1779
1780     my $vmid = $1;
1781
1782     my $conf = $res;
1783     my $descr = '';
1784
1785     my @lines = split(/\n/, $raw);
1786     foreach my $line (@lines) {
1787         next if $line =~ m/^\s*$/;
1788
1789         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1790             my $snapname = $1;
1791             $conf->{description} = $descr if $descr;
1792             $descr = '';
1793             $conf = $res->{snapshots}->{$snapname} = {};
1794             next;
1795         }
1796
1797         if ($line =~ m/^\#(.*)\s*$/) {
1798             $descr .= PVE::Tools::decode_text($1) . "\n";
1799             next;
1800         }
1801
1802         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1803             $descr .= PVE::Tools::decode_text($2);
1804         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1805             $conf->{snapstate} = $1;
1806         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1807             my $key = $1;
1808             my $value = $2;
1809             $conf->{$key} = $value;
1810         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1811             my $key = $1;
1812             my $value = $2;
1813             eval { $value = check_type($key, $value); };
1814             if ($@) {
1815                 warn "vm $vmid - unable to parse value of '$key' - $@";
1816             } else {
1817                 my $fmt = $confdesc->{$key}->{format};
1818                 if ($fmt && $fmt eq 'pve-qm-drive') {
1819                     my $v = parse_drive($key, $value);
1820                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1821                         $v->{file} = $volid;
1822                         $value = print_drive($vmid, $v);
1823                     } else {
1824                         warn "vm $vmid - unable to parse value of '$key'\n";
1825                         next;
1826                     }
1827                 }
1828
1829                 if ($key eq 'cdrom') {
1830                     $conf->{ide2} = $value;
1831                 } else {
1832                     $conf->{$key} = $value;
1833                 }
1834             }
1835         }
1836     }
1837
1838     $conf->{description} = $descr if $descr;
1839
1840     delete $res->{snapstate}; # just to be sure
1841
1842     return $res;
1843 }
1844
1845 sub write_vm_config {
1846     my ($filename, $conf) = @_;
1847
1848     delete $conf->{snapstate}; # just to be sure
1849
1850     if ($conf->{cdrom}) {
1851         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
1852         $conf->{ide2} = $conf->{cdrom};
1853         delete $conf->{cdrom};
1854     }
1855
1856     # we do not use 'smp' any longer
1857     if ($conf->{sockets}) {
1858         delete $conf->{smp};
1859     } elsif ($conf->{smp}) {
1860         $conf->{sockets} = $conf->{smp};
1861         delete $conf->{cores};
1862         delete $conf->{smp};
1863     }
1864
1865     if ($conf->{maxcpus} && $conf->{sockets}) {
1866         delete $conf->{sockets};
1867     }
1868
1869     my $used_volids = {};
1870
1871     my $cleanup_config = sub {
1872         my ($cref, $snapname) = @_;
1873
1874         foreach my $key (keys %$cref) {
1875             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
1876                 $key eq 'snapstate';
1877             my $value = $cref->{$key};
1878             eval { $value = check_type($key, $value); };
1879             die "unable to parse value of '$key' - $@" if $@;
1880
1881             $cref->{$key} = $value;
1882
1883             if (!$snapname && valid_drivename($key)) {
1884                 my $drive = parse_drive($key, $value);
1885                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
1886             }
1887         }
1888     };
1889
1890     &$cleanup_config($conf);
1891     foreach my $snapname (keys %{$conf->{snapshots}}) {
1892         &$cleanup_config($conf->{snapshots}->{$snapname}, $snapname);
1893     }
1894
1895     # remove 'unusedX' settings if we re-add a volume
1896     foreach my $key (keys %$conf) {
1897         my $value = $conf->{$key};
1898         if ($key =~ m/^unused/ && $used_volids->{$value}) {
1899             delete $conf->{$key};
1900         }
1901     }
1902
1903     my $generate_raw_config = sub {
1904         my ($conf) = @_;
1905
1906         my $raw = '';
1907
1908         # add description as comment to top of file
1909         my $descr = $conf->{description} || '';
1910         foreach my $cl (split(/\n/, $descr)) {
1911             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
1912         }
1913
1914         foreach my $key (sort keys %$conf) {
1915             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots';
1916             $raw .= "$key: $conf->{$key}\n";
1917         }
1918         return $raw;
1919     };
1920
1921     my $raw = &$generate_raw_config($conf);
1922     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
1923         $raw .= "\n[$snapname]\n";
1924         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
1925     }
1926
1927     return $raw;
1928 }
1929
1930 sub update_config_nolock {
1931     my ($vmid, $conf, $skiplock) = @_;
1932
1933     check_lock($conf) if !$skiplock;
1934
1935     my $cfspath = cfs_config_path($vmid);
1936
1937     PVE::Cluster::cfs_write_file($cfspath, $conf);
1938 }
1939
1940 sub update_config {
1941     my ($vmid, $conf, $skiplock) = @_;
1942
1943     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
1944 }
1945
1946 sub load_defaults {
1947
1948     my $res = {};
1949
1950     # we use static defaults from our JSON schema configuration
1951     foreach my $key (keys %$confdesc) {
1952         if (defined(my $default = $confdesc->{$key}->{default})) {
1953             $res->{$key} = $default;
1954         }
1955     }
1956
1957     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
1958     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
1959
1960     return $res;
1961 }
1962
1963 sub config_list {
1964     my $vmlist = PVE::Cluster::get_vmlist();
1965     my $res = {};
1966     return $res if !$vmlist || !$vmlist->{ids};
1967     my $ids = $vmlist->{ids};
1968
1969     foreach my $vmid (keys %$ids) {
1970         my $d = $ids->{$vmid};
1971         next if !$d->{node} || $d->{node} ne $nodename;
1972         next if !$d->{type} || $d->{type} ne 'qemu';
1973         $res->{$vmid}->{exists} = 1;
1974     }
1975     return $res;
1976 }
1977
1978 # test if VM uses local resources (to prevent migration)
1979 sub check_local_resources {
1980     my ($conf, $noerr) = @_;
1981
1982     my $loc_res = 0;
1983
1984     $loc_res = 1 if $conf->{hostusb}; # old syntax
1985     $loc_res = 1 if $conf->{hostpci}; # old syntax
1986
1987     foreach my $k (keys %$conf) {
1988         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
1989         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
1990     }
1991
1992     die "VM uses local resources\n" if $loc_res && !$noerr;
1993
1994     return $loc_res;
1995 }
1996
1997 # check if used storages are available on all nodes (use by migrate)
1998 sub check_storage_availability {
1999     my ($storecfg, $conf, $node) = @_;
2000
2001     foreach_drive($conf, sub {
2002         my ($ds, $drive) = @_;
2003
2004         my $volid = $drive->{file};
2005         return if !$volid;
2006
2007         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2008         return if !$sid;
2009
2010         # check if storage is available on both nodes
2011         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2012         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2013    });
2014 }
2015
2016 # list nodes where all VM images are available (used by has_feature API)
2017 sub shared_nodes {
2018     my ($conf, $storecfg) = @_;
2019
2020     my $nodelist = PVE::Cluster::get_nodelist();
2021     my $nodehash = { map { $_ => 1 } @$nodelist };
2022     my $nodename = PVE::INotify::nodename();
2023
2024     foreach_drive($conf, sub {
2025         my ($ds, $drive) = @_;
2026
2027         my $volid = $drive->{file};
2028         return if !$volid;
2029
2030         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2031         if ($storeid) {
2032             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2033             if ($scfg->{disable}) {
2034                 $nodehash = {};
2035             } elsif (my $avail = $scfg->{nodes}) {
2036                 foreach my $node (keys %$nodehash) {
2037                     delete $nodehash->{$node} if !$avail->{$node};
2038                 }
2039             } elsif (!$scfg->{shared}) {
2040                 foreach my $node (keys %$nodehash) {
2041                     delete $nodehash->{$node} if $node ne $nodename
2042                 }
2043             }
2044         }
2045     });
2046
2047     return $nodehash
2048 }
2049
2050 sub check_lock {
2051     my ($conf) = @_;
2052
2053     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2054 }
2055
2056 sub check_cmdline {
2057     my ($pidfile, $pid) = @_;
2058
2059     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2060     if (defined($fh)) {
2061         my $line = <$fh>;
2062         $fh->close;
2063         return undef if !$line;
2064         my @param = split(/\0/, $line);
2065
2066         my $cmd = $param[0];
2067         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2068
2069         for (my $i = 0; $i < scalar (@param); $i++) {
2070             my $p = $param[$i];
2071             next if !$p;
2072             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2073                 my $p = $param[$i+1];
2074                 return 1 if $p && ($p eq $pidfile);
2075                 return undef;
2076             }
2077         }
2078     }
2079     return undef;
2080 }
2081
2082 sub check_running {
2083     my ($vmid, $nocheck, $node) = @_;
2084
2085     my $filename = config_file($vmid, $node);
2086
2087     die "unable to find configuration file for VM $vmid - no such machine\n"
2088         if !$nocheck && ! -f $filename;
2089
2090     my $pidfile = pidfile_name($vmid);
2091
2092     if (my $fd = IO::File->new("<$pidfile")) {
2093         my $st = stat($fd);
2094         my $line = <$fd>;
2095         close($fd);
2096
2097         my $mtime = $st->mtime;
2098         if ($mtime > time()) {
2099             warn "file '$filename' modified in future\n";
2100         }
2101
2102         if ($line =~ m/^(\d+)$/) {
2103             my $pid = $1;
2104             if (check_cmdline($pidfile, $pid)) {
2105                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2106                     return $pid;
2107                 }
2108             }
2109         }
2110     }
2111
2112     return undef;
2113 }
2114
2115 sub vzlist {
2116
2117     my $vzlist = config_list();
2118
2119     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2120
2121     while (defined(my $de = $fd->read)) {
2122         next if $de !~ m/^(\d+)\.pid$/;
2123         my $vmid = $1;
2124         next if !defined($vzlist->{$vmid});
2125         if (my $pid = check_running($vmid)) {
2126             $vzlist->{$vmid}->{pid} = $pid;
2127         }
2128     }
2129
2130     return $vzlist;
2131 }
2132
2133 sub disksize {
2134     my ($storecfg, $conf) = @_;
2135
2136     my $bootdisk = $conf->{bootdisk};
2137     return undef if !$bootdisk;
2138     return undef if !valid_drivename($bootdisk);
2139
2140     return undef if !$conf->{$bootdisk};
2141
2142     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2143     return undef if !defined($drive);
2144
2145     return undef if drive_is_cdrom($drive);
2146
2147     my $volid = $drive->{file};
2148     return undef if !$volid;
2149
2150     return $drive->{size};
2151 }
2152
2153 my $last_proc_pid_stat;
2154
2155 # get VM status information
2156 # This must be fast and should not block ($full == false)
2157 # We only query KVM using QMP if $full == true (this can be slow)
2158 sub vmstatus {
2159     my ($opt_vmid, $full) = @_;
2160
2161     my $res = {};
2162
2163     my $storecfg = PVE::Storage::config();
2164
2165     my $list = vzlist();
2166     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2167
2168     my $cpucount = $cpuinfo->{cpus} || 1;
2169
2170     foreach my $vmid (keys %$list) {
2171         next if $opt_vmid && ($vmid ne $opt_vmid);
2172
2173         my $cfspath = cfs_config_path($vmid);
2174         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2175
2176         my $d = {};
2177         $d->{pid} = $list->{$vmid}->{pid};
2178
2179         # fixme: better status?
2180         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2181
2182         my $size = disksize($storecfg, $conf);
2183         if (defined($size)) {
2184             $d->{disk} = 0; # no info available
2185             $d->{maxdisk} = $size;
2186         } else {
2187             $d->{disk} = 0;
2188             $d->{maxdisk} = 0;
2189         }
2190
2191         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2192         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2193
2194         $d->{name} = $conf->{name} || "VM $vmid";
2195         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2196
2197         if ($conf->{balloon}) {
2198             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2199             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2200         }
2201
2202         $d->{uptime} = 0;
2203         $d->{cpu} = 0;
2204         $d->{mem} = 0;
2205
2206         $d->{netout} = 0;
2207         $d->{netin} = 0;
2208
2209         $d->{diskread} = 0;
2210         $d->{diskwrite} = 0;
2211
2212         $d->{template} = is_template($conf);
2213
2214         $res->{$vmid} = $d;
2215     }
2216
2217     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2218     foreach my $dev (keys %$netdev) {
2219         next if $dev !~ m/^tap([1-9]\d*)i/;
2220         my $vmid = $1;
2221         my $d = $res->{$vmid};
2222         next if !$d;
2223
2224         $d->{netout} += $netdev->{$dev}->{receive};
2225         $d->{netin} += $netdev->{$dev}->{transmit};
2226     }
2227
2228     my $ctime = gettimeofday;
2229
2230     foreach my $vmid (keys %$list) {
2231
2232         my $d = $res->{$vmid};
2233         my $pid = $d->{pid};
2234         next if !$pid;
2235
2236         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2237         next if !$pstat; # not running
2238
2239         my $used = $pstat->{utime} + $pstat->{stime};
2240
2241         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2242
2243         if ($pstat->{vsize}) {
2244             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2245         }
2246
2247         my $old = $last_proc_pid_stat->{$pid};
2248         if (!$old) {
2249             $last_proc_pid_stat->{$pid} = {
2250                 time => $ctime,
2251                 used => $used,
2252                 cpu => 0,
2253             };
2254             next;
2255         }
2256
2257         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2258
2259         if ($dtime > 1000) {
2260             my $dutime = $used -  $old->{used};
2261
2262             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2263             $last_proc_pid_stat->{$pid} = {
2264                 time => $ctime,
2265                 used => $used,
2266                 cpu => $d->{cpu},
2267             };
2268         } else {
2269             $d->{cpu} = $old->{cpu};
2270         }
2271     }
2272
2273     return $res if !$full;
2274
2275     my $qmpclient = PVE::QMPClient->new();
2276
2277     my $ballooncb = sub {
2278         my ($vmid, $resp) = @_;
2279
2280         my $info = $resp->{'return'};
2281         return if !$info->{max_mem};
2282
2283         my $d = $res->{$vmid};
2284
2285         # use memory assigned to VM
2286         $d->{maxmem} = $info->{max_mem};
2287         $d->{balloon} = $info->{actual};
2288
2289         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2290             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2291             $d->{freemem} = $info->{free_mem};
2292         }
2293
2294     };
2295
2296     my $blockstatscb = sub {
2297         my ($vmid, $resp) = @_;
2298         my $data = $resp->{'return'} || [];
2299         my $totalrdbytes = 0;
2300         my $totalwrbytes = 0;
2301         for my $blockstat (@$data) {
2302             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2303             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2304         }
2305         $res->{$vmid}->{diskread} = $totalrdbytes;
2306         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2307     };
2308
2309     my $statuscb = sub {
2310         my ($vmid, $resp) = @_;
2311
2312         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2313         # this fails if ballon driver is not loaded, so this must be
2314         # the last commnand (following command are aborted if this fails).
2315         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2316
2317         my $status = 'unknown';
2318         if (!defined($status = $resp->{'return'}->{status})) {
2319             warn "unable to get VM status\n";
2320             return;
2321         }
2322
2323         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2324     };
2325
2326     foreach my $vmid (keys %$list) {
2327         next if $opt_vmid && ($vmid ne $opt_vmid);
2328         next if !$res->{$vmid}->{pid}; # not running
2329         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2330     }
2331
2332     $qmpclient->queue_execute();
2333
2334     foreach my $vmid (keys %$list) {
2335         next if $opt_vmid && ($vmid ne $opt_vmid);
2336         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2337     }
2338
2339     return $res;
2340 }
2341
2342 sub foreach_drive {
2343     my ($conf, $func) = @_;
2344
2345     foreach my $ds (keys %$conf) {
2346         next if !valid_drivename($ds);
2347
2348         my $drive = parse_drive($ds, $conf->{$ds});
2349         next if !$drive;
2350
2351         &$func($ds, $drive);
2352     }
2353 }
2354
2355 sub foreach_volid {
2356     my ($conf, $func) = @_;
2357
2358     my $volhash = {};
2359
2360     my $test_volid = sub {
2361         my ($volid, $is_cdrom) = @_;
2362
2363         return if !$volid;
2364
2365         $volhash->{$volid} = $is_cdrom || 0;
2366     };
2367
2368     foreach_drive($conf, sub {
2369         my ($ds, $drive) = @_;
2370         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2371     });
2372
2373     foreach my $snapname (keys %{$conf->{snapshots}}) {
2374         my $snap = $conf->{snapshots}->{$snapname};
2375         &$test_volid($snap->{vmstate}, 0);
2376         foreach_drive($snap, sub {
2377             my ($ds, $drive) = @_;
2378             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2379         });
2380     }
2381
2382     foreach my $volid (keys %$volhash) {
2383         &$func($volid, $volhash->{$volid});
2384     }
2385 }
2386
2387 sub vga_conf_has_spice {
2388     my ($vga) = @_;
2389
2390     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2391
2392     return $1 || 1;
2393 }
2394
2395 sub config_to_command {
2396     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2397
2398     my $cmd = [];
2399     my $globalFlags = [];
2400     my $machineFlags = [];
2401     my $rtcFlags = [];
2402     my $cpuFlags = [];
2403     my $devices = [];
2404     my $pciaddr = '';
2405     my $bridges = {};
2406     my $kvmver = kvm_user_version();
2407     my $vernum = 0; # unknown
2408     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2409         $vernum = $1*1000000+$2*1000;
2410     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2411         $vernum = $1*1000000+$2*1000+$3;
2412     }
2413
2414     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2415
2416     my $have_ovz = -f '/proc/vz/vestat';
2417
2418     my $q35 = machine_type_is_q35($conf);
2419
2420     push @$cmd, '/usr/bin/kvm';
2421
2422     push @$cmd, '-id', $vmid;
2423
2424     my $use_virtio = 0;
2425
2426     my $qmpsocket = qmp_socket($vmid);
2427     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2428     push @$cmd, '-mon', "chardev=qmp,mode=control";
2429
2430     my $socket = vnc_socket($vmid);
2431     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2432
2433     push @$cmd, '-pidfile' , pidfile_name($vmid);
2434
2435     push @$cmd, '-daemonize';
2436
2437     if ($conf->{smbios1}) {
2438         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2439     }
2440
2441     if ($q35) {
2442         # the q35 chipset support native usb2, so we enable usb controller
2443         # by default for this machine type
2444         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2445     } else {
2446         $pciaddr = print_pci_addr("piix3", $bridges);
2447         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2448
2449         my $use_usb2 = 0;
2450         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2451             next if !$conf->{"usb$i"};
2452             $use_usb2 = 1;
2453         }
2454         # include usb device config
2455         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2456     }
2457
2458     my $vga = $conf->{vga};
2459
2460     my $qxlnum = vga_conf_has_spice($vga);
2461     $vga = 'qxl' if $qxlnum;
2462
2463     if (!$vga) {
2464         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2465                                 $conf->{ostype} eq 'win7' ||
2466                                 $conf->{ostype} eq 'w2k8')) {
2467             $vga = 'std';
2468         } else {
2469             $vga = 'cirrus';
2470         }
2471     }
2472
2473     # enable absolute mouse coordinates (needed by vnc)
2474     my $tablet;
2475     if (defined($conf->{tablet})) {
2476         $tablet = $conf->{tablet};
2477     } else {
2478         $tablet = $defaults->{tablet};
2479         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2480         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2481     }
2482
2483     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2484
2485     # host pci devices
2486     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2487         my $d = parse_hostpci($conf->{"hostpci$i"});
2488         next if !$d;
2489
2490         my $pcie = $d->{pcie};
2491         if($pcie){
2492             die "q35 machine model is not enabled" if !$q35;
2493             $pciaddr = print_pcie_addr("hostpci$i");
2494         }else{
2495             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2496         }
2497
2498         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2499         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2500         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2501         if ($xvga && $xvga ne '') {
2502             push @$cpuFlags, 'kvm=off';
2503             $vga = 'none';
2504         }
2505         $driver = "vfio-pci" if $xvga ne '';
2506         my $pcidevices = $d->{pciid};
2507         my $multifunction = 1 if @$pcidevices > 1;
2508
2509         my $j=0;
2510         foreach my $pcidevice (@$pcidevices) {
2511
2512             my $id = "hostpci$i";
2513             $id .= ".$j" if $multifunction;
2514             my $addr = $pciaddr;
2515             $addr .= ".$j" if $multifunction;
2516             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2517
2518             if($j == 0){
2519                 $devicestr .= "$rombar$xvga";
2520                 $devicestr .= ",multifunction=on" if $multifunction;
2521             }
2522
2523             push @$devices, '-device', $devicestr;
2524             $j++;
2525         }
2526     }
2527
2528     # usb devices
2529     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2530         my $d = parse_usb_device($conf->{"usb$i"});
2531         next if !$d;
2532         if ($d->{vendorid} && $d->{productid}) {
2533             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2534         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2535             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2536         } elsif ($d->{spice}) {
2537             # usb redir support for spice
2538             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2539             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2540         }
2541     }
2542
2543     # serial devices
2544     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2545         if (my $path = $conf->{"serial$i"}) {
2546             if ($path eq 'socket') {
2547                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2548                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2549                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2550             } else {
2551                 die "no such serial device\n" if ! -c $path;
2552                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2553                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2554             }
2555         }
2556     }
2557
2558     # parallel devices
2559     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2560         if (my $path = $conf->{"parallel$i"}) {
2561             die "no such parallel device\n" if ! -c $path;
2562             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2563             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2564             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2565         }
2566     }
2567
2568     my $vmname = $conf->{name} || "vm$vmid";
2569
2570     push @$cmd, '-name', $vmname;
2571
2572     my $sockets = 1;
2573     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2574     $sockets = $conf->{sockets} if  $conf->{sockets};
2575
2576     my $cores = $conf->{cores} || 1;
2577     my $maxcpus = $conf->{maxcpus} if $conf->{maxcpus};
2578
2579     if ($maxcpus) {
2580         push @$cmd, '-smp', "cpus=$cores,maxcpus=$maxcpus";
2581     } else {
2582         push @$cmd, '-smp', "sockets=$sockets,cores=$cores";
2583     }
2584
2585     push @$cmd, '-nodefaults';
2586
2587     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2588
2589     my $bootindex_hash = {};
2590     my $i = 1;
2591     foreach my $o (split(//, $bootorder)) {
2592         $bootindex_hash->{$o} = $i*100;
2593         $i++;
2594     }
2595
2596     push @$cmd, '-boot', "menu=on";
2597
2598     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2599
2600     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2601
2602     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2603
2604     # time drift fix
2605     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2606
2607     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2608     my $useLocaltime = $conf->{localtime};
2609
2610     if (my $ost = $conf->{ostype}) {
2611         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2612
2613         if ($ost =~ m/^w/) { # windows
2614             $useLocaltime = 1 if !defined($conf->{localtime});
2615
2616             # use time drift fix when acpi is enabled
2617             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2618                 $tdf = 1 if !defined($conf->{tdf});
2619             }
2620         }
2621
2622         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2623             $ost eq 'wvista') {
2624             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2625             push @$cmd, '-no-hpet';
2626             #push @$cpuFlags , 'hv_vapic" if !$nokvm;  #fixme, my win2008R2 hang at boot with this
2627             push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2628         }
2629
2630         if ($ost eq 'win7' || $ost eq 'win8') {
2631             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2632         }
2633     }
2634
2635     push @$rtcFlags, 'driftfix=slew' if $tdf;
2636
2637     if ($nokvm) {
2638         push @$machineFlags, 'accel=tcg';
2639     } else {
2640         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2641     }
2642
2643     my $machine_type = $forcemachine || $conf->{machine};
2644     if ($machine_type) {
2645         push @$machineFlags, "type=${machine_type}";
2646     }
2647
2648     if ($conf->{startdate}) {
2649         push @$rtcFlags, "base=$conf->{startdate}";
2650     } elsif ($useLocaltime) {
2651         push @$rtcFlags, 'base=localtime';
2652     }
2653
2654     my $cpu = $nokvm ? "qemu64" : "kvm64";
2655     $cpu = $conf->{cpu} if $conf->{cpu};
2656
2657     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2658
2659     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2660
2661     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2662
2663     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2664
2665     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2666
2667     # Note: enforce needs kernel 3.10, so we do not use it for now
2668     # push @$cmd, '-cpu', "$cpu,enforce";
2669     push @$cmd, '-cpu', $cpu;
2670
2671     push @$cmd, '-S' if $conf->{freeze};
2672
2673     # set keyboard layout
2674     my $kb = $conf->{keyboard} || $defaults->{keyboard};
2675     push @$cmd, '-k', $kb if $kb;
2676
2677     # enable sound
2678     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
2679     #push @$cmd, '-soundhw', 'es1370';
2680     #push @$cmd, '-soundhw', $soundhw if $soundhw;
2681
2682     if($conf->{agent}) {
2683         my $qgasocket = qga_socket($vmid);
2684         my $pciaddr = print_pci_addr("qga0", $bridges);
2685         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
2686         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
2687         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
2688     }
2689
2690     my $spice_port;
2691
2692     if ($qxlnum) {
2693         if ($qxlnum > 1) {
2694             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
2695                 for(my $i = 1; $i < $qxlnum; $i++){
2696                     my $pciaddr = print_pci_addr("vga$i", $bridges);
2697                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
2698                 }
2699             } else {
2700                 # assume other OS works like Linux
2701                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
2702                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
2703             }
2704         }
2705
2706         my $pciaddr = print_pci_addr("spice", $bridges);
2707
2708         $spice_port = PVE::Tools::next_spice_port();
2709
2710         push @$devices, '-spice', "tls-port=${spice_port},addr=127.0.0.1,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
2711
2712         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
2713         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
2714         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
2715     }
2716
2717     # enable balloon by default, unless explicitly disabled
2718     if (!defined($conf->{balloon}) || $conf->{balloon}) {
2719         $pciaddr = print_pci_addr("balloon0", $bridges);
2720         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
2721     }
2722
2723     if ($conf->{watchdog}) {
2724         my $wdopts = parse_watchdog($conf->{watchdog});
2725         $pciaddr = print_pci_addr("watchdog", $bridges);
2726         my $watchdog = $wdopts->{model} || 'i6300esb';
2727         push @$devices, '-device', "$watchdog$pciaddr";
2728         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
2729     }
2730
2731     my $vollist = [];
2732     my $scsicontroller = {};
2733     my $ahcicontroller = {};
2734     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
2735
2736     # Add iscsi initiator name if available
2737     if (my $initiator = get_initiator_name()) {
2738         push @$devices, '-iscsi', "initiator-name=$initiator";
2739     }
2740
2741     foreach_drive($conf, sub {
2742         my ($ds, $drive) = @_;
2743
2744         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
2745             push @$vollist, $drive->{file};
2746         }
2747
2748         $use_virtio = 1 if $ds =~ m/^virtio/;
2749
2750         if (drive_is_cdrom ($drive)) {
2751             if ($bootindex_hash->{d}) {
2752                 $drive->{bootindex} = $bootindex_hash->{d};
2753                 $bootindex_hash->{d} += 1;
2754             }
2755         } else {
2756             if ($bootindex_hash->{c}) {
2757                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
2758                 $bootindex_hash->{c} += 1;
2759             }
2760         }
2761
2762         if ($drive->{interface} eq 'scsi') {
2763
2764             my $maxdev = ($scsihw !~ m/^lsi/) ? 256 : 7;
2765             my $controller = int($drive->{index} / $maxdev);
2766             $pciaddr = print_pci_addr("scsihw$controller", $bridges);
2767             push @$devices, '-device', "$scsihw,id=scsihw$controller$pciaddr" if !$scsicontroller->{$controller};
2768             $scsicontroller->{$controller}=1;
2769         }
2770
2771         if ($drive->{interface} eq 'sata') {
2772            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
2773            $pciaddr = print_pci_addr("ahci$controller", $bridges);
2774            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
2775            $ahcicontroller->{$controller}=1;
2776         }
2777
2778         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
2779         push @$devices, '-drive',$drive_cmd;
2780         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
2781     });
2782
2783     push @$cmd, '-m', $conf->{memory} || $defaults->{memory};
2784
2785     for (my $i = 0; $i < $MAX_NETS; $i++) {
2786          next if !$conf->{"net$i"};
2787          my $d = parse_net($conf->{"net$i"});
2788          next if !$d;
2789
2790          $use_virtio = 1 if $d->{model} eq 'virtio';
2791
2792          if ($bootindex_hash->{n}) {
2793             $d->{bootindex} = $bootindex_hash->{n};
2794             $bootindex_hash->{n} += 1;
2795          }
2796
2797          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
2798          push @$devices, '-netdev', $netdevfull;
2799
2800          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
2801          push @$devices, '-device', $netdevicefull;
2802     }
2803
2804     if (!$q35) {
2805         # add pci bridges
2806         while (my ($k, $v) = each %$bridges) {
2807             $pciaddr = print_pci_addr("pci.$k");
2808             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
2809         }
2810     }
2811
2812     # hack: virtio with fairsched is unreliable, so we do not use fairsched
2813     # when the VM uses virtio devices.
2814     if (!$use_virtio && $have_ovz) {
2815
2816         my $cpuunits = defined($conf->{cpuunits}) ?
2817             $conf->{cpuunits} : $defaults->{cpuunits};
2818
2819         push @$cmd, '-cpuunits', $cpuunits if $cpuunits;
2820
2821         # fixme: cpulimit is currently ignored
2822         #push @$cmd, '-cpulimit', $conf->{cpulimit} if $conf->{cpulimit};
2823     }
2824
2825     # add custom args
2826     if ($conf->{args}) {
2827         my $aa = PVE::Tools::split_args($conf->{args});
2828         push @$cmd, @$aa;
2829     }
2830
2831     push @$cmd, @$devices;
2832     push @$cmd, '-rtc', join(',', @$rtcFlags)
2833         if scalar(@$rtcFlags);
2834     push @$cmd, '-machine', join(',', @$machineFlags)
2835         if scalar(@$machineFlags);
2836     push @$cmd, '-global', join(',', @$globalFlags)
2837         if scalar(@$globalFlags);
2838
2839     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
2840 }
2841
2842 sub vnc_socket {
2843     my ($vmid) = @_;
2844     return "${var_run_tmpdir}/$vmid.vnc";
2845 }
2846
2847 sub spice_port {
2848     my ($vmid) = @_;
2849
2850     my $res = vm_mon_cmd($vmid, 'query-spice');
2851
2852     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
2853 }
2854
2855 sub qmp_socket {
2856     my ($vmid) = @_;
2857     return "${var_run_tmpdir}/$vmid.qmp";
2858 }
2859
2860 sub qga_socket {
2861     my ($vmid) = @_;
2862     return "${var_run_tmpdir}/$vmid.qga";
2863 }
2864
2865 sub pidfile_name {
2866     my ($vmid) = @_;
2867     return "${var_run_tmpdir}/$vmid.pid";
2868 }
2869
2870 sub vm_devices_list {
2871     my ($vmid) = @_;
2872
2873     my $res = vm_mon_cmd($vmid, 'query-pci');
2874
2875     my $devices = {};
2876     foreach my $pcibus (@$res) {
2877         foreach my $device (@{$pcibus->{devices}}) {
2878             next if !$device->{'qdev_id'};
2879             $devices->{$device->{'qdev_id'}} = 1;
2880         }
2881     }
2882
2883     my $resblock = vm_mon_cmd($vmid, 'query-block');
2884     foreach my $block (@$resblock) {
2885         if($block->{device} =~ m/^drive-(\S+)/){
2886                 $devices->{$1} = 1;
2887         }
2888     }
2889
2890     return $devices;
2891 }
2892
2893 sub vm_deviceplug {
2894     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
2895
2896     return 1 if !check_running($vmid);
2897
2898     my $q35 = machine_type_is_q35($conf);
2899
2900     if ($deviceid eq 'tablet') {
2901         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
2902         return 1;
2903     }
2904
2905     return 1 if !$conf->{hotplug};
2906
2907     my $devices_list = vm_devices_list($vmid);
2908     return 1 if defined($devices_list->{$deviceid});
2909
2910     qemu_bridgeadd($storecfg, $conf, $vmid, $deviceid); #add bridge if we need it for the device
2911
2912     if ($deviceid =~ m/^(virtio)(\d+)$/) {
2913         return undef if !qemu_driveadd($storecfg, $vmid, $device);
2914         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
2915         qemu_deviceadd($vmid, $devicefull);
2916         if(!qemu_deviceaddverify($vmid, $deviceid)) {
2917            qemu_drivedel($vmid, $deviceid);
2918            return undef;
2919         }
2920     }
2921
2922     if ($deviceid =~ m/^(scsihw)(\d+)$/) {
2923         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
2924         my $pciaddr = print_pci_addr($deviceid);
2925         my $devicefull = "$scsihw,id=$deviceid$pciaddr";
2926         qemu_deviceadd($vmid, $devicefull);
2927         return undef if(!qemu_deviceaddverify($vmid, $deviceid));
2928     }
2929
2930     if ($deviceid =~ m/^(scsi)(\d+)$/) {
2931         return undef if !qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
2932         return undef if !qemu_driveadd($storecfg, $vmid, $device);
2933         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
2934         if(!qemu_deviceadd($vmid, $devicefull)) {
2935            qemu_drivedel($vmid, $deviceid);
2936            return undef;
2937         }
2938     }
2939
2940     if ($deviceid =~ m/^(net)(\d+)$/) {
2941         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
2942         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
2943         qemu_deviceadd($vmid, $netdevicefull);
2944         if(!qemu_deviceaddverify($vmid, $deviceid)) {
2945            qemu_netdevdel($vmid, $deviceid);
2946            return undef;
2947         }
2948     }
2949
2950
2951     if (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
2952         my $bridgeid = $2;
2953         my $pciaddr = print_pci_addr($deviceid);
2954         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
2955         qemu_deviceadd($vmid, $devicefull);
2956         return undef if !qemu_deviceaddverify($vmid, $deviceid);
2957     }
2958
2959     return 1;
2960 }
2961
2962 sub vm_deviceunplug {
2963     my ($vmid, $conf, $deviceid) = @_;
2964
2965     return 1 if !check_running ($vmid);
2966
2967     if ($deviceid eq 'tablet') {
2968         qemu_devicedel($vmid, $deviceid);
2969         return 1;
2970     }
2971
2972     return 1 if !$conf->{hotplug};
2973
2974     my $devices_list = vm_devices_list($vmid);
2975     return 1 if !defined($devices_list->{$deviceid});
2976
2977     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
2978
2979     if ($deviceid =~ m/^(virtio)(\d+)$/) {
2980         qemu_devicedel($vmid, $deviceid);
2981         return undef if !qemu_devicedelverify($vmid, $deviceid);
2982         return undef if !qemu_drivedel($vmid, $deviceid);
2983     }
2984
2985     if ($deviceid =~ m/^(lsi)(\d+)$/) {
2986         return undef if !qemu_devicedel($vmid, $deviceid);
2987     }
2988
2989     if ($deviceid =~ m/^(scsi)(\d+)$/) {
2990         return undef if !qemu_devicedel($vmid, $deviceid);
2991         return undef if !qemu_drivedel($vmid, $deviceid);
2992     }
2993
2994     if ($deviceid =~ m/^(net)(\d+)$/) {
2995         qemu_devicedel($vmid, $deviceid);
2996         return undef if !qemu_devicedelverify($vmid, $deviceid);
2997         return undef if !qemu_netdevdel($vmid, $deviceid);
2998     }
2999
3000     return 1;
3001 }
3002
3003 sub qemu_deviceadd {
3004     my ($vmid, $devicefull) = @_;
3005
3006     $devicefull = "driver=".$devicefull;
3007     my %options =  split(/[=,]/, $devicefull);
3008
3009     vm_mon_cmd($vmid, "device_add" , %options);
3010     return 1;
3011 }
3012
3013 sub qemu_devicedel {
3014     my($vmid, $deviceid) = @_;
3015     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3016     return 1;
3017 }
3018
3019 sub qemu_driveadd {
3020     my($storecfg, $vmid, $device) = @_;
3021
3022     my $drive = print_drive_full($storecfg, $vmid, $device);
3023     my $ret = vm_human_monitor_command($vmid, "drive_add auto $drive");
3024     # If the command succeeds qemu prints: "OK"
3025     if ($ret !~ m/OK/s) {
3026         syslog("err", "adding drive failed: $ret");
3027         return undef;
3028     }
3029     return 1;
3030 }
3031
3032 sub qemu_drivedel {
3033     my($vmid, $deviceid) = @_;
3034
3035     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3036     $ret =~ s/^\s+//;
3037     if ($ret =~ m/Device \'.*?\' not found/s) {
3038         # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3039     }
3040     elsif ($ret ne "") {
3041       syslog("err", "deleting drive $deviceid failed : $ret");
3042       return undef;
3043     }
3044     return 1;
3045 }
3046
3047 sub qemu_deviceaddverify {
3048     my ($vmid,$deviceid) = @_;
3049
3050     for (my $i = 0; $i <= 5; $i++) {
3051          my $devices_list = vm_devices_list($vmid);
3052          return 1 if defined($devices_list->{$deviceid});
3053          sleep 1;
3054     }
3055     syslog("err", "error on hotplug device $deviceid");
3056     return undef;
3057 }
3058
3059
3060 sub qemu_devicedelverify {
3061     my ($vmid,$deviceid) = @_;
3062
3063     #need to verify the device is correctly remove as device_del is async and empty return is not reliable
3064     for (my $i = 0; $i <= 5; $i++) {
3065          my $devices_list = vm_devices_list($vmid);
3066          return 1 if !defined($devices_list->{$deviceid});
3067          sleep 1;
3068     }
3069     syslog("err", "error on hot-unplugging device $deviceid");
3070     return undef;
3071 }
3072
3073 sub qemu_findorcreatescsihw {
3074     my ($storecfg, $conf, $vmid, $device) = @_;
3075
3076     my $maxdev = ($conf->{scsihw} && ($conf->{scsihw} !~ m/^lsi/)) ? 256 : 7;
3077     my $controller = int($device->{index} / $maxdev);
3078     my $scsihwid="scsihw$controller";
3079     my $devices_list = vm_devices_list($vmid);
3080
3081     if(!defined($devices_list->{$scsihwid})) {
3082        return undef if !vm_deviceplug($storecfg, $conf, $vmid, $scsihwid);
3083     }
3084     return 1;
3085 }
3086
3087 sub qemu_bridgeadd {
3088     my ($storecfg, $conf, $vmid, $device) = @_;
3089
3090     my $bridges = {};
3091     my $bridgeid = undef;
3092     print_pci_addr($device, $bridges);
3093
3094     while (my ($k, $v) = each %$bridges) {
3095         $bridgeid = $k;
3096     }
3097     return if !$bridgeid || $bridgeid < 1;
3098     my $bridge = "pci.$bridgeid";
3099     my $devices_list = vm_devices_list($vmid);
3100
3101     if(!defined($devices_list->{$bridge})) {
3102         return undef if !vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3103     }
3104     return 1;
3105 }
3106
3107 sub qemu_netdevadd {
3108     my ($vmid, $conf, $device, $deviceid) = @_;
3109
3110     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3111     my %options =  split(/[=,]/, $netdev);
3112
3113     vm_mon_cmd($vmid, "netdev_add",  %options);
3114     return 1;
3115 }
3116
3117 sub qemu_netdevdel {
3118     my ($vmid, $deviceid) = @_;
3119
3120     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3121     return 1;
3122 }
3123
3124 sub qemu_cpu_hotplug {
3125     my ($vmid, $conf, $cores) = @_;
3126
3127     die "new cores config is not defined" if !$cores;
3128     die "you can't add more cores than maxcpus"
3129         if $conf->{maxcpus} && ($cores > $conf->{maxcpus});
3130     return if !check_running($vmid);
3131
3132     my $currentcores = $conf->{cores} if $conf->{cores};
3133     die "current cores is not defined" if !$currentcores;
3134     die "maxcpus is not defined" if !$conf->{maxcpus};
3135     raise_param_exc({ 'cores' => "online cpu unplug is not yet possible" })
3136         if($cores < $currentcores);
3137
3138     my $currentrunningcores = vm_mon_cmd($vmid, "query-cpus");
3139     raise_param_exc({ 'cores' => "cores number if running vm is different than configuration" })
3140         if scalar (@{$currentrunningcores}) != $currentcores;
3141
3142     for(my $i = $currentcores; $i < $cores; $i++) {
3143         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3144     }
3145 }
3146
3147 sub qemu_block_set_io_throttle {
3148     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3149
3150     return if !check_running($vmid) ;
3151
3152     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3153
3154 }
3155
3156 # old code, only used to shutdown old VM after update
3157 sub __read_avail {
3158     my ($fh, $timeout) = @_;
3159
3160     my $sel = new IO::Select;
3161     $sel->add($fh);
3162
3163     my $res = '';
3164     my $buf;
3165
3166     my @ready;
3167     while (scalar (@ready = $sel->can_read($timeout))) {
3168         my $count;
3169         if ($count = $fh->sysread($buf, 8192)) {
3170             if ($buf =~ /^(.*)\(qemu\) $/s) {
3171                 $res .= $1;
3172                 last;
3173             } else {
3174                 $res .= $buf;
3175             }
3176         } else {
3177             if (!defined($count)) {
3178                 die "$!\n";
3179             }
3180             last;
3181         }
3182     }
3183
3184     die "monitor read timeout\n" if !scalar(@ready);
3185
3186     return $res;
3187 }
3188
3189 # old code, only used to shutdown old VM after update
3190 sub vm_monitor_command {
3191     my ($vmid, $cmdstr, $nocheck) = @_;
3192
3193     my $res;
3194
3195     eval {
3196         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3197
3198         my $sname = "${var_run_tmpdir}/$vmid.mon";
3199
3200         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3201             die "unable to connect to VM $vmid socket - $!\n";
3202
3203         my $timeout = 3;
3204
3205         # hack: migrate sometime blocks the monitor (when migrate_downtime
3206         # is set)
3207         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3208             $timeout = 60*60; # 1 hour
3209         }
3210
3211         # read banner;
3212         my $data = __read_avail($sock, $timeout);
3213
3214         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3215             die "got unexpected qemu monitor banner\n";
3216         }
3217
3218         my $sel = new IO::Select;
3219         $sel->add($sock);
3220
3221         if (!scalar(my @ready = $sel->can_write($timeout))) {
3222             die "monitor write error - timeout";
3223         }
3224
3225         my $fullcmd = "$cmdstr\r";
3226
3227         # syslog('info', "VM $vmid monitor command: $cmdstr");
3228
3229         my $b;
3230         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3231             die "monitor write error - $!";
3232         }
3233
3234         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3235
3236         $timeout = 20;
3237
3238         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3239             $timeout = 60*60; # 1 hour
3240         } elsif ($cmdstr =~ m/^(eject|change)/) {
3241             $timeout = 60; # note: cdrom mount command is slow
3242         }
3243         if ($res = __read_avail($sock, $timeout)) {
3244
3245             my @lines = split("\r?\n", $res);
3246
3247             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3248
3249             $res = join("\n", @lines);
3250             $res .= "\n";
3251         }
3252     };
3253
3254     my $err = $@;
3255
3256     if ($err) {
3257         syslog("err", "VM $vmid monitor command failed - $err");
3258         die $err;
3259     }
3260
3261     return $res;
3262 }
3263
3264 sub qemu_block_resize {
3265     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3266
3267     my $running = check_running($vmid);
3268
3269     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3270
3271     return if !$running;
3272
3273     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3274
3275 }
3276
3277 sub qemu_volume_snapshot {
3278     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3279
3280     my $running = check_running($vmid);
3281
3282     return if !PVE::Storage::volume_snapshot($storecfg, $volid, $snap, $running);
3283
3284     return if !$running;
3285
3286     vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3287
3288 }
3289
3290 sub qemu_volume_snapshot_delete {
3291     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3292
3293     my $running = check_running($vmid);
3294
3295     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3296
3297     return if !$running;
3298
3299     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3300 }
3301
3302 sub qga_freezefs {
3303     my ($vmid) = @_;
3304
3305     #need to impplement call to qemu-ga
3306 }
3307
3308 sub qga_unfreezefs {
3309     my ($vmid) = @_;
3310
3311     #need to impplement call to qemu-ga
3312 }
3313
3314 sub set_migration_caps {
3315     my ($vmid) = @_;
3316
3317     my $cap_ref = [];
3318
3319     my $enabled_cap = {
3320         "auto-converge" => 1,
3321         "xbzrle" => 0,
3322         "x-rdma-pin-all" => 0,
3323         "zero-blocks" => 0,
3324     };
3325
3326     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3327
3328     for my $supported_capability (@$supported_capabilities) {
3329         push @$cap_ref, {
3330             capability => $supported_capability->{capability},
3331             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3332         };
3333     }
3334
3335     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3336 }
3337
3338 sub vm_start {
3339     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
3340
3341     lock_config($vmid, sub {
3342         my $conf = load_config($vmid, $migratedfrom);
3343
3344         die "you can't start a vm if it's a template\n" if is_template($conf);
3345
3346         check_lock($conf) if !$skiplock;
3347
3348         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
3349
3350         my $defaults = load_defaults();
3351
3352         # set environment variable useful inside network script
3353         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
3354
3355         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
3356
3357         my $migrate_port = 0;
3358         my $migrate_uri;
3359         if ($statefile) {
3360             if ($statefile eq 'tcp') {
3361                 my $localip = "localhost";
3362                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
3363                 if ($datacenterconf->{migration_unsecure}) {
3364                         my $nodename = PVE::INotify::nodename();
3365                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
3366                 }
3367                 $migrate_port = PVE::Tools::next_migrate_port();
3368                 $migrate_uri = "tcp:${localip}:${migrate_port}";
3369                 push @$cmd, '-incoming', $migrate_uri;
3370                 push @$cmd, '-S';
3371             } else {
3372                 push @$cmd, '-loadstate', $statefile;
3373             }
3374         } elsif ($paused) {
3375             push @$cmd, '-S';
3376         }
3377
3378         # host pci devices
3379         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3380           my $d = parse_hostpci($conf->{"hostpci$i"});
3381           next if !$d;
3382           my $pcidevices = $d->{pciid};
3383           foreach my $pcidevice (@$pcidevices) {
3384                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
3385
3386                 my $info = pci_device_info("0000:$pciid");
3387                 die "IOMMU not present\n" if !check_iommu_support();
3388                 die "no pci device info for device '$pciid'\n" if !$info;
3389
3390                 if ($d->{driver} && $d->{driver} eq "vfio") {
3391                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
3392                 } else {
3393                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
3394                 }
3395
3396                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
3397           }
3398         }
3399
3400         PVE::Storage::activate_volumes($storecfg, $vollist);
3401
3402         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
3403                     umask => 0077); };
3404         my $err = $@;
3405         die "start failed: $err" if $err;
3406
3407         print "migration listens on $migrate_uri\n" if $migrate_uri;
3408
3409         if ($statefile && $statefile ne 'tcp')  {
3410             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
3411             warn $@ if $@;
3412         }
3413
3414         if ($migratedfrom) {
3415
3416             eval {
3417                 PVE::QemuServer::set_migration_caps($vmid);
3418             };
3419             warn $@ if $@;
3420
3421             if ($spice_port) {
3422                 print "spice listens on port $spice_port\n";
3423                 if ($spice_ticket) {
3424                     PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
3425                     PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
3426                 }
3427             }
3428
3429         } else {
3430
3431             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
3432                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
3433                     if $conf->{balloon};
3434                 vm_mon_cmd_nocheck($vmid, 'qom-set',
3435                             path => "machine/peripheral/balloon0",
3436                             property => "guest-stats-polling-interval",
3437                             value => 2);
3438             }
3439         }
3440     });
3441 }
3442
3443 sub vm_mon_cmd {
3444     my ($vmid, $execute, %params) = @_;
3445
3446     my $cmd = { execute => $execute, arguments => \%params };
3447     vm_qmp_command($vmid, $cmd);
3448 }
3449
3450 sub vm_mon_cmd_nocheck {
3451     my ($vmid, $execute, %params) = @_;
3452
3453     my $cmd = { execute => $execute, arguments => \%params };
3454     vm_qmp_command($vmid, $cmd, 1);
3455 }
3456
3457 sub vm_qmp_command {
3458     my ($vmid, $cmd, $nocheck) = @_;
3459
3460     my $res;
3461
3462     my $timeout;
3463     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
3464         $timeout = $cmd->{arguments}->{timeout};
3465         delete $cmd->{arguments}->{timeout};
3466     }
3467
3468     eval {
3469         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3470         my $sname = qmp_socket($vmid);
3471         if (-e $sname) {
3472             my $qmpclient = PVE::QMPClient->new();
3473
3474             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
3475         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
3476             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
3477                 if scalar(%{$cmd->{arguments}});
3478             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
3479         } else {
3480             die "unable to open monitor socket\n";
3481         }
3482     };
3483     if (my $err = $@) {
3484         syslog("err", "VM $vmid qmp command failed - $err");
3485         die $err;
3486     }
3487
3488     return $res;
3489 }
3490
3491 sub vm_human_monitor_command {
3492     my ($vmid, $cmdline) = @_;
3493
3494     my $res;
3495
3496     my $cmd = {
3497         execute => 'human-monitor-command',
3498         arguments => { 'command-line' => $cmdline},
3499     };
3500
3501     return vm_qmp_command($vmid, $cmd);
3502 }
3503
3504 sub vm_commandline {
3505     my ($storecfg, $vmid) = @_;
3506
3507     my $conf = load_config($vmid);
3508
3509     my $defaults = load_defaults();
3510
3511     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
3512
3513     return join(' ', @$cmd);
3514 }
3515
3516 sub vm_reset {
3517     my ($vmid, $skiplock) = @_;
3518
3519     lock_config($vmid, sub {
3520
3521         my $conf = load_config($vmid);
3522
3523         check_lock($conf) if !$skiplock;
3524
3525         vm_mon_cmd($vmid, "system_reset");
3526     });
3527 }
3528
3529 sub get_vm_volumes {
3530     my ($conf) = @_;
3531
3532     my $vollist = [];
3533     foreach_volid($conf, sub {
3534         my ($volid, $is_cdrom) = @_;
3535
3536         return if $volid =~ m|^/|;
3537
3538         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3539         return if !$sid;
3540
3541         push @$vollist, $volid;
3542     });
3543
3544     return $vollist;
3545 }
3546
3547 sub vm_stop_cleanup {
3548     my ($storecfg, $vmid, $conf, $keepActive) = @_;
3549
3550     eval {
3551         fairsched_rmnod($vmid); # try to destroy group
3552
3553         if (!$keepActive) {
3554             my $vollist = get_vm_volumes($conf);
3555             PVE::Storage::deactivate_volumes($storecfg, $vollist);
3556         }
3557
3558         foreach my $ext (qw(mon qmp pid vnc qga)) {
3559             unlink "/var/run/qemu-server/${vmid}.$ext";
3560         }
3561     };
3562     warn $@ if $@; # avoid errors - just warn
3563 }
3564
3565 # Note: use $nockeck to skip tests if VM configuration file exists.
3566 # We need that when migration VMs to other nodes (files already moved)
3567 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
3568 sub vm_stop {
3569     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
3570
3571     $force = 1 if !defined($force) && !$shutdown;
3572
3573     if ($migratedfrom){
3574         my $pid = check_running($vmid, $nocheck, $migratedfrom);
3575         kill 15, $pid if $pid;
3576         my $conf = load_config($vmid, $migratedfrom);
3577         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive);
3578         return;
3579     }
3580
3581     lock_config($vmid, sub {
3582
3583         my $pid = check_running($vmid, $nocheck);
3584         return if !$pid;
3585
3586         my $conf;
3587         if (!$nocheck) {
3588             $conf = load_config($vmid);
3589             check_lock($conf) if !$skiplock;
3590             if (!defined($timeout) && $shutdown && $conf->{startup}) {
3591                 my $opts = parse_startup($conf->{startup});
3592                 $timeout = $opts->{down} if $opts->{down};
3593             }
3594         }
3595
3596         $timeout = 60 if !defined($timeout);
3597
3598         eval {
3599             if ($shutdown) {
3600                 $nocheck ? vm_mon_cmd_nocheck($vmid, "system_powerdown") : vm_mon_cmd($vmid, "system_powerdown");
3601
3602             } else {
3603                 $nocheck ? vm_mon_cmd_nocheck($vmid, "quit") : vm_mon_cmd($vmid, "quit");
3604             }
3605         };
3606         my $err = $@;
3607
3608         if (!$err) {
3609             my $count = 0;
3610             while (($count < $timeout) && check_running($vmid, $nocheck)) {
3611                 $count++;
3612                 sleep 1;
3613             }
3614
3615             if ($count >= $timeout) {
3616                 if ($force) {
3617                     warn "VM still running - terminating now with SIGTERM\n";
3618                     kill 15, $pid;
3619                 } else {
3620                     die "VM quit/powerdown failed - got timeout\n";
3621                 }
3622             } else {
3623                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive) if $conf;
3624                 return;
3625             }
3626         } else {
3627             if ($force) {
3628                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
3629                 kill 15, $pid;
3630             } else {
3631                 die "VM quit/powerdown failed\n";
3632             }
3633         }
3634
3635         # wait again
3636         $timeout = 10;
3637
3638         my $count = 0;
3639         while (($count < $timeout) && check_running($vmid, $nocheck)) {
3640             $count++;
3641             sleep 1;
3642         }
3643
3644         if ($count >= $timeout) {
3645             warn "VM still running - terminating now with SIGKILL\n";
3646             kill 9, $pid;
3647             sleep 1;
3648         }
3649
3650         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive) if $conf;
3651    });
3652 }
3653
3654 sub vm_suspend {
3655     my ($vmid, $skiplock) = @_;
3656
3657     lock_config($vmid, sub {
3658
3659         my $conf = load_config($vmid);
3660
3661         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
3662
3663         vm_mon_cmd($vmid, "stop");
3664     });
3665 }
3666
3667 sub vm_resume {
3668     my ($vmid, $skiplock) = @_;
3669
3670     lock_config($vmid, sub {
3671
3672         my $conf = load_config($vmid);
3673
3674         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
3675
3676         vm_mon_cmd($vmid, "cont");
3677     });
3678 }
3679
3680 sub vm_sendkey {
3681     my ($vmid, $skiplock, $key) = @_;
3682
3683     lock_config($vmid, sub {
3684
3685         my $conf = load_config($vmid);
3686
3687         # there is no qmp command, so we use the human monitor command
3688         vm_human_monitor_command($vmid, "sendkey $key");
3689     });
3690 }
3691
3692 sub vm_destroy {
3693     my ($storecfg, $vmid, $skiplock) = @_;
3694
3695     lock_config($vmid, sub {
3696
3697         my $conf = load_config($vmid);
3698
3699         check_lock($conf) if !$skiplock;
3700
3701         if (!check_running($vmid)) {
3702             fairsched_rmnod($vmid); # try to destroy group
3703             destroy_vm($storecfg, $vmid);
3704         } else {
3705             die "VM $vmid is running - destroy failed\n";
3706         }
3707     });
3708 }
3709
3710 # pci helpers
3711
3712 sub file_write {
3713     my ($filename, $buf) = @_;
3714
3715     my $fh = IO::File->new($filename, "w");
3716     return undef if !$fh;
3717
3718     my $res = print $fh $buf;
3719
3720     $fh->close();
3721
3722     return $res;
3723 }
3724
3725 sub pci_device_info {
3726     my ($name) = @_;
3727
3728     my $res;
3729
3730     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
3731     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
3732
3733     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
3734     return undef if !defined($irq) || $irq !~ m/^\d+$/;
3735
3736     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
3737     return undef if !defined($vendor) || $vendor !~ s/^0x//;
3738
3739     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
3740     return undef if !defined($product) || $product !~ s/^0x//;
3741
3742     $res = {
3743         name => $name,
3744         vendor => $vendor,
3745         product => $product,
3746         domain => $domain,
3747         bus => $bus,
3748         slot => $slot,
3749         func => $func,
3750         irq => $irq,
3751         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
3752     };
3753
3754     return $res;
3755 }
3756
3757 sub pci_dev_reset {
3758     my ($dev) = @_;
3759
3760     my $name = $dev->{name};
3761
3762     my $fn = "$pcisysfs/devices/$name/reset";
3763
3764     return file_write($fn, "1");
3765 }
3766
3767 sub pci_dev_bind_to_stub {
3768     my ($dev) = @_;
3769
3770     my $name = $dev->{name};
3771
3772     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
3773     return 1 if -d $testdir;
3774
3775     my $data = "$dev->{vendor} $dev->{product}";
3776     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
3777
3778     my $fn = "$pcisysfs/devices/$name/driver/unbind";
3779     if (!file_write($fn, $name)) {
3780         return undef if -f $fn;
3781     }
3782
3783     $fn = "$pcisysfs/drivers/pci-stub/bind";
3784     if (! -d $testdir) {
3785         return undef if !file_write($fn, $name);
3786     }
3787
3788     return -d $testdir;
3789 }
3790
3791 sub pci_dev_bind_to_vfio {
3792     my ($dev) = @_;
3793
3794     my $name = $dev->{name};
3795
3796     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
3797
3798     if (!-d $vfio_basedir) {
3799         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
3800     }
3801     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
3802
3803     my $testdir = "$vfio_basedir/$name";
3804     return 1 if -d $testdir;
3805
3806     my $data = "$dev->{vendor} $dev->{product}";
3807     return undef if !file_write("$vfio_basedir/new_id", $data);
3808
3809     my $fn = "$pcisysfs/devices/$name/driver/unbind";
3810     if (!file_write($fn, $name)) {
3811         return undef if -f $fn;
3812     }
3813
3814     $fn = "$vfio_basedir/bind";
3815     if (! -d $testdir) {
3816         return undef if !file_write($fn, $name);
3817     }
3818
3819     return -d $testdir;
3820 }
3821
3822 sub pci_dev_group_bind_to_vfio {
3823     my ($pciid) = @_;
3824
3825     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
3826
3827     if (!-d $vfio_basedir) {
3828         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
3829     }
3830     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
3831
3832     # get IOMMU group devices
3833     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
3834       my @devs = grep /^0000:/, readdir($D);
3835     closedir($D);
3836
3837     foreach my $pciid (@devs) {
3838         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
3839         my $info = pci_device_info($1);
3840         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
3841     }
3842
3843     return 1;
3844 }
3845
3846 sub print_pci_addr {
3847     my ($id, $bridges) = @_;
3848
3849     my $res = '';
3850     my $devices = {
3851         piix3 => { bus => 0, addr => 1 },
3852         #addr2 : first videocard
3853         balloon0 => { bus => 0, addr => 3 },
3854         watchdog => { bus => 0, addr => 4 },
3855         scsihw0 => { bus => 0, addr => 5 },
3856         scsihw1 => { bus => 0, addr => 6 },
3857         ahci0 => { bus => 0, addr => 7 },
3858         qga0 => { bus => 0, addr => 8 },
3859         spice => { bus => 0, addr => 9 },
3860         virtio0 => { bus => 0, addr => 10 },
3861         virtio1 => { bus => 0, addr => 11 },
3862         virtio2 => { bus => 0, addr => 12 },
3863         virtio3 => { bus => 0, addr => 13 },
3864         virtio4 => { bus => 0, addr => 14 },
3865         virtio5 => { bus => 0, addr => 15 },
3866         hostpci0 => { bus => 0, addr => 16 },
3867         hostpci1 => { bus => 0, addr => 17 },
3868         net0 => { bus => 0, addr => 18 },
3869         net1 => { bus => 0, addr => 19 },
3870         net2 => { bus => 0, addr => 20 },
3871         net3 => { bus => 0, addr => 21 },
3872         net4 => { bus => 0, addr => 22 },
3873         net5 => { bus => 0, addr => 23 },
3874         vga1 => { bus => 0, addr => 24 },
3875         vga2 => { bus => 0, addr => 25 },
3876         vga3 => { bus => 0, addr => 26 },
3877         hostpci2 => { bus => 0, addr => 27 },
3878         hostpci3 => { bus => 0, addr => 28 },
3879         #addr29 : usb-host (pve-usb.cfg)
3880         'pci.1' => { bus => 0, addr => 30 },
3881         'pci.2' => { bus => 0, addr => 31 },
3882         'net6' => { bus => 1, addr => 1 },
3883         'net7' => { bus => 1, addr => 2 },
3884         'net8' => { bus => 1, addr => 3 },
3885         'net9' => { bus => 1, addr => 4 },
3886         'net10' => { bus => 1, addr => 5 },
3887         'net11' => { bus => 1, addr => 6 },
3888         'net12' => { bus => 1, addr => 7 },
3889         'net13' => { bus => 1, addr => 8 },
3890         'net14' => { bus => 1, addr => 9 },
3891         'net15' => { bus => 1, addr => 10 },
3892         'net16' => { bus => 1, addr => 11 },
3893         'net17' => { bus => 1, addr => 12 },
3894         'net18' => { bus => 1, addr => 13 },
3895         'net19' => { bus => 1, addr => 14 },
3896         'net20' => { bus => 1, addr => 15 },
3897         'net21' => { bus => 1, addr => 16 },
3898         'net22' => { bus => 1, addr => 17 },
3899         'net23' => { bus => 1, addr => 18 },
3900         'net24' => { bus => 1, addr => 19 },
3901         'net25' => { bus => 1, addr => 20 },
3902         'net26' => { bus => 1, addr => 21 },
3903         'net27' => { bus => 1, addr => 22 },
3904         'net28' => { bus => 1, addr => 23 },
3905         'net29' => { bus => 1, addr => 24 },
3906         'net30' => { bus => 1, addr => 25 },
3907         'net31' => { bus => 1, addr => 26 },
3908         'virtio6' => { bus => 2, addr => 1 },
3909         'virtio7' => { bus => 2, addr => 2 },
3910         'virtio8' => { bus => 2, addr => 3 },
3911         'virtio9' => { bus => 2, addr => 4 },
3912         'virtio10' => { bus => 2, addr => 5 },
3913         'virtio11' => { bus => 2, addr => 6 },
3914         'virtio12' => { bus => 2, addr => 7 },
3915         'virtio13' => { bus => 2, addr => 8 },
3916         'virtio14' => { bus => 2, addr => 9 },
3917         'virtio15' => { bus => 2, addr => 10 },
3918     };
3919
3920     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
3921            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
3922            my $bus = $devices->{$id}->{bus};
3923            $res = ",bus=pci.$bus,addr=$addr";
3924            $bridges->{$bus} = 1 if $bridges;
3925     }
3926     return $res;
3927
3928 }
3929
3930 sub print_pcie_addr {
3931     my ($id) = @_;
3932
3933     my $res = '';
3934     my $devices = {
3935         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
3936         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
3937         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
3938         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
3939     };
3940
3941     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
3942            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
3943            my $bus = $devices->{$id}->{bus};
3944            $res = ",bus=$bus,addr=$addr";
3945     }
3946     return $res;
3947
3948 }
3949
3950 # vzdump restore implementaion
3951
3952 sub tar_archive_read_firstfile {
3953     my $archive = shift;
3954
3955     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
3956
3957     # try to detect archive type first
3958     my $pid = open (TMP, "tar tf '$archive'|") ||
3959         die "unable to open file '$archive'\n";
3960     my $firstfile = <TMP>;
3961     kill 15, $pid;
3962     close TMP;
3963
3964     die "ERROR: archive contaions no data\n" if !$firstfile;
3965     chomp $firstfile;
3966
3967     return $firstfile;
3968 }
3969
3970 sub tar_restore_cleanup {
3971     my ($storecfg, $statfile) = @_;
3972
3973     print STDERR "starting cleanup\n";
3974
3975     if (my $fd = IO::File->new($statfile, "r")) {
3976         while (defined(my $line = <$fd>)) {
3977             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
3978                 my $volid = $2;
3979                 eval {
3980                     if ($volid =~ m|^/|) {
3981                         unlink $volid || die 'unlink failed\n';
3982                     } else {
3983                         PVE::Storage::vdisk_free($storecfg, $volid);
3984                     }
3985                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
3986                 };
3987                 print STDERR "unable to cleanup '$volid' - $@" if $@;
3988             } else {
3989                 print STDERR "unable to parse line in statfile - $line";
3990             }
3991         }
3992         $fd->close();
3993     }
3994 }
3995
3996 sub restore_archive {
3997     my ($archive, $vmid, $user, $opts) = @_;
3998
3999     my $format = $opts->{format};
4000     my $comp;
4001
4002     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4003         $format = 'tar' if !$format;
4004         $comp = 'gzip';
4005     } elsif ($archive =~ m/\.tar$/) {
4006         $format = 'tar' if !$format;
4007     } elsif ($archive =~ m/.tar.lzo$/) {
4008         $format = 'tar' if !$format;
4009         $comp = 'lzop';
4010     } elsif ($archive =~ m/\.vma$/) {
4011         $format = 'vma' if !$format;
4012     } elsif ($archive =~ m/\.vma\.gz$/) {
4013         $format = 'vma' if !$format;
4014         $comp = 'gzip';
4015     } elsif ($archive =~ m/\.vma\.lzo$/) {
4016         $format = 'vma' if !$format;
4017         $comp = 'lzop';
4018     } else {
4019         $format = 'vma' if !$format; # default
4020     }
4021
4022     # try to detect archive format
4023     if ($format eq 'tar') {
4024         return restore_tar_archive($archive, $vmid, $user, $opts);
4025     } else {
4026         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4027     }
4028 }
4029
4030 sub restore_update_config_line {
4031     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4032
4033     return if $line =~ m/^\#qmdump\#/;
4034     return if $line =~ m/^\#vzdump\#/;
4035     return if $line =~ m/^lock:/;
4036     return if $line =~ m/^unused\d+:/;
4037     return if $line =~ m/^parent:/;
4038     return if $line =~ m/^template:/; # restored VM is never a template
4039
4040     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4041         # try to convert old 1.X settings
4042         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4043         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4044             my ($model, $macaddr) = split(/\=/, $devconfig);
4045             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4046             my $net = {
4047                 model => $model,
4048                 bridge => "vmbr$ind",
4049                 macaddr => $macaddr,
4050             };
4051             my $netstr = print_net($net);
4052
4053             print $outfd "net$cookie->{netcount}: $netstr\n";
4054             $cookie->{netcount}++;
4055         }
4056     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4057         my ($id, $netstr) = ($1, $2);
4058         my $net = parse_net($netstr);
4059         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4060         $netstr = print_net($net);
4061         print $outfd "$id: $netstr\n";
4062     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4063         my $virtdev = $1;
4064         my $value = $3;
4065         if ($line =~ m/backup=no/) {
4066             print $outfd "#$line";
4067         } elsif ($virtdev && $map->{$virtdev}) {
4068             my $di = parse_drive($virtdev, $value);
4069             delete $di->{format}; # format can change on restore
4070             $di->{file} = $map->{$virtdev};
4071             $value = print_drive($vmid, $di);
4072             print $outfd "$virtdev: $value\n";
4073         } else {
4074             print $outfd $line;
4075         }
4076     } else {
4077         print $outfd $line;
4078     }
4079 }
4080
4081 sub scan_volids {
4082     my ($cfg, $vmid) = @_;
4083
4084     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4085
4086     my $volid_hash = {};
4087     foreach my $storeid (keys %$info) {
4088         foreach my $item (@{$info->{$storeid}}) {
4089             next if !($item->{volid} && $item->{size});
4090             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4091             $volid_hash->{$item->{volid}} = $item;
4092         }
4093     }
4094
4095     return $volid_hash;
4096 }
4097
4098 sub get_used_paths {
4099     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
4100
4101     my $used_path = {};
4102
4103     my $scan_config = sub {
4104         my ($cref, $snapname) = @_;
4105
4106         foreach my $key (keys %$cref) {
4107             my $value = $cref->{$key};
4108             if (valid_drivename($key)) {
4109                 next if $skip_drive && $key eq $skip_drive;
4110                 my $drive = parse_drive($key, $value);
4111                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
4112                 if ($drive->{file} =~ m!^/!) {
4113                     $used_path->{$drive->{file}}++; # = 1;
4114                 } else {
4115                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
4116                     next if !$storeid;
4117                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
4118                     next if !$scfg;
4119                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
4120                     $used_path->{$path}++; # = 1;
4121                 }
4122             }
4123         }
4124     };
4125
4126     &$scan_config($conf);
4127
4128     undef $skip_drive;
4129
4130     if ($scan_snapshots) {
4131         foreach my $snapname (keys %{$conf->{snapshots}}) {
4132             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
4133         }
4134     }
4135
4136     return $used_path;
4137 }
4138
4139 sub update_disksize {
4140     my ($vmid, $conf, $volid_hash) = @_;
4141
4142     my $changes;
4143
4144     my $used = {};
4145
4146     # Note: it is allowed to define multiple storages with same path (alias), so
4147     # we need to check both 'volid' and real 'path' (two different volid can point
4148     # to the same path).
4149
4150     my $usedpath = {};
4151
4152     # update size info
4153     foreach my $opt (keys %$conf) {
4154         if (valid_drivename($opt)) {
4155             my $drive = parse_drive($opt, $conf->{$opt});
4156             my $volid = $drive->{file};
4157             next if !$volid;
4158
4159             $used->{$volid} = 1;
4160             if ($volid_hash->{$volid} &&
4161                 (my $path = $volid_hash->{$volid}->{path})) {
4162                 $usedpath->{$path} = 1;
4163             }
4164
4165             next if drive_is_cdrom($drive);
4166             next if !$volid_hash->{$volid};
4167
4168             $drive->{size} = $volid_hash->{$volid}->{size};
4169             my $new = print_drive($vmid, $drive);
4170             if ($new ne $conf->{$opt}) {
4171                 $changes = 1;
4172                 $conf->{$opt} = $new;
4173             }
4174         }
4175     }
4176
4177     # remove 'unusedX' entry if volume is used
4178     foreach my $opt (keys %$conf) {
4179         next if $opt !~ m/^unused\d+$/;
4180         my $volid = $conf->{$opt};
4181         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
4182         if ($used->{$volid} || ($path && $usedpath->{$path})) {
4183             $changes = 1;
4184             delete $conf->{$opt};
4185         }
4186     }
4187
4188     foreach my $volid (sort keys %$volid_hash) {
4189         next if $volid =~ m/vm-$vmid-state-/;
4190         next if $used->{$volid};
4191         my $path = $volid_hash->{$volid}->{path};
4192         next if !$path; # just to be sure
4193         next if $usedpath->{$path};
4194         $changes = 1;
4195         add_unused_volume($conf, $volid);
4196         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
4197     }
4198
4199     return $changes;
4200 }
4201
4202 sub rescan {
4203     my ($vmid, $nolock) = @_;
4204
4205     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
4206
4207     my $volid_hash = scan_volids($cfg, $vmid);
4208
4209     my $updatefn =  sub {
4210         my ($vmid) = @_;
4211
4212         my $conf = load_config($vmid);
4213
4214         check_lock($conf);
4215
4216         my $vm_volids = {};
4217         foreach my $volid (keys %$volid_hash) {
4218             my $info = $volid_hash->{$volid};
4219             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
4220         }
4221
4222         my $changes = update_disksize($vmid, $conf, $vm_volids);
4223
4224         update_config_nolock($vmid, $conf, 1) if $changes;
4225     };
4226
4227     if (defined($vmid)) {
4228         if ($nolock) {
4229             &$updatefn($vmid);
4230         } else {
4231             lock_config($vmid, $updatefn, $vmid);
4232         }
4233     } else {
4234         my $vmlist = config_list();
4235         foreach my $vmid (keys %$vmlist) {
4236             if ($nolock) {
4237                 &$updatefn($vmid);
4238             } else {
4239                 lock_config($vmid, $updatefn, $vmid);
4240             }
4241         }
4242     }
4243 }
4244
4245 sub restore_vma_archive {
4246     my ($archive, $vmid, $user, $opts, $comp) = @_;
4247
4248     my $input = $archive eq '-' ? "<&STDIN" : undef;
4249     my $readfrom = $archive;
4250
4251     my $uncomp = '';
4252     if ($comp) {
4253         $readfrom = '-';
4254         my $qarchive = PVE::Tools::shellquote($archive);
4255         if ($comp eq 'gzip') {
4256             $uncomp = "zcat $qarchive|";
4257         } elsif ($comp eq 'lzop') {
4258             $uncomp = "lzop -d -c $qarchive|";
4259         } else {
4260             die "unknown compression method '$comp'\n";
4261         }
4262
4263     }
4264
4265     my $tmpdir = "/var/tmp/vzdumptmp$$";
4266     rmtree $tmpdir;
4267
4268     # disable interrupts (always do cleanups)
4269     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
4270         warn "got interrupt - ignored\n";
4271     };
4272
4273     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
4274     POSIX::mkfifo($mapfifo, 0600);
4275     my $fifofh;
4276
4277     my $openfifo = sub {
4278         open($fifofh, '>', $mapfifo) || die $!;
4279     };
4280
4281     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
4282
4283     my $oldtimeout;
4284     my $timeout = 5;
4285
4286     my $devinfo = {};
4287
4288     my $rpcenv = PVE::RPCEnvironment::get();
4289
4290     my $conffile = config_file($vmid);
4291     my $tmpfn = "$conffile.$$.tmp";
4292
4293     # Note: $oldconf is undef if VM does not exists
4294     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
4295
4296     my $print_devmap = sub {
4297         my $virtdev_hash = {};
4298
4299         my $cfgfn = "$tmpdir/qemu-server.conf";
4300
4301         # we can read the config - that is already extracted
4302         my $fh = IO::File->new($cfgfn, "r") ||
4303             "unable to read qemu-server.conf - $!\n";
4304
4305         while (defined(my $line = <$fh>)) {
4306             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
4307                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
4308                 die "archive does not contain data for drive '$virtdev'\n"
4309                     if !$devinfo->{$devname};
4310                 if (defined($opts->{storage})) {
4311                     $storeid = $opts->{storage} || 'local';
4312                 } elsif (!$storeid) {
4313                     $storeid = 'local';
4314                 }
4315                 $format = 'raw' if !$format;
4316                 $devinfo->{$devname}->{devname} = $devname;
4317                 $devinfo->{$devname}->{virtdev} = $virtdev;
4318                 $devinfo->{$devname}->{format} = $format;
4319                 $devinfo->{$devname}->{storeid} = $storeid;
4320
4321                 # check permission on storage
4322                 my $pool = $opts->{pool}; # todo: do we need that?
4323                 if ($user ne 'root@pam') {
4324                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
4325                 }
4326
4327                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
4328             }
4329         }
4330
4331         foreach my $devname (keys %$devinfo) {
4332             die "found no device mapping information for device '$devname'\n"
4333                 if !$devinfo->{$devname}->{virtdev};
4334         }
4335
4336         my $cfg = cfs_read_file('storage.cfg');
4337
4338         # create empty/temp config
4339         if ($oldconf) {
4340             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
4341             foreach_drive($oldconf, sub {
4342                 my ($ds, $drive) = @_;
4343
4344                 return if drive_is_cdrom($drive);
4345
4346                 my $volid = $drive->{file};
4347
4348                 return if !$volid || $volid =~ m|^/|;
4349
4350                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
4351                 return if !$path || !$owner || ($owner != $vmid);
4352
4353                 # Note: only delete disk we want to restore
4354                 # other volumes will become unused
4355                 if ($virtdev_hash->{$ds}) {
4356                     PVE::Storage::vdisk_free($cfg, $volid);
4357                 }
4358             });
4359         }
4360
4361         my $map = {};
4362         foreach my $virtdev (sort keys %$virtdev_hash) {
4363             my $d = $virtdev_hash->{$virtdev};
4364             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
4365             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
4366
4367             # test if requested format is supported
4368             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
4369             my $supported = grep { $_ eq $d->{format} } @$validFormats;
4370             $d->{format} = $defFormat if !$supported;
4371
4372             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
4373                                                   $d->{format}, undef, $alloc_size);
4374             print STDERR "new volume ID is '$volid'\n";
4375             $d->{volid} = $volid;
4376             my $path = PVE::Storage::path($cfg, $volid);
4377
4378             my $write_zeros = 1;
4379             # fixme: what other storages types initialize volumes with zero?
4380             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
4381                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
4382                 $write_zeros = 0;
4383             }
4384
4385             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
4386
4387             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
4388             $map->{$virtdev} = $volid;
4389         }
4390
4391         $fh->seek(0, 0) || die "seek failed - $!\n";
4392
4393         my $outfd = new IO::File ($tmpfn, "w") ||
4394             die "unable to write config for VM $vmid\n";
4395
4396         my $cookie = { netcount => 0 };
4397         while (defined(my $line = <$fh>)) {
4398             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
4399         }
4400
4401         $fh->close();
4402         $outfd->close();
4403     };
4404
4405     eval {
4406         # enable interrupts
4407         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
4408             die "interrupted by signal\n";
4409         };
4410         local $SIG{ALRM} = sub { die "got timeout\n"; };
4411
4412         $oldtimeout = alarm($timeout);
4413
4414         my $parser = sub {
4415             my $line = shift;
4416
4417             print "$line\n";
4418
4419             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
4420                 my ($dev_id, $size, $devname) = ($1, $2, $3);
4421                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
4422             } elsif ($line =~ m/^CTIME: /) {
4423                 # we correctly received the vma config, so we can disable
4424                 # the timeout now for disk allocation (set to 10 minutes, so
4425                 # that we always timeout if something goes wrong)
4426                 alarm(600);
4427                 &$print_devmap();
4428                 print $fifofh "done\n";
4429                 my $tmp = $oldtimeout || 0;
4430                 $oldtimeout = undef;
4431                 alarm($tmp);
4432                 close($fifofh);
4433             }
4434         };
4435
4436         print "restore vma archive: $cmd\n";
4437         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
4438     };
4439     my $err = $@;
4440
4441     alarm($oldtimeout) if $oldtimeout;
4442
4443     unlink $mapfifo;
4444
4445     if ($err) {
4446         rmtree $tmpdir;
4447         unlink $tmpfn;
4448
4449         my $cfg = cfs_read_file('storage.cfg');
4450         foreach my $devname (keys %$devinfo) {
4451             my $volid = $devinfo->{$devname}->{volid};
4452             next if !$volid;
4453             eval {
4454                 if ($volid =~ m|^/|) {
4455                     unlink $volid || die 'unlink failed\n';
4456                 } else {
4457                     PVE::Storage::vdisk_free($cfg, $volid);
4458                 }
4459                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
4460             };
4461             print STDERR "unable to cleanup '$volid' - $@" if $@;
4462         }
4463         die $err;
4464     }
4465
4466     rmtree $tmpdir;
4467
4468     rename($tmpfn, $conffile) ||
4469         die "unable to commit configuration file '$conffile'\n";
4470
4471     PVE::Cluster::cfs_update(); # make sure we read new file
4472
4473     eval { rescan($vmid, 1); };
4474     warn $@ if $@;
4475 }
4476
4477 sub restore_tar_archive {
4478     my ($archive, $vmid, $user, $opts) = @_;
4479
4480     if ($archive ne '-') {
4481         my $firstfile = tar_archive_read_firstfile($archive);
4482         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
4483             if $firstfile ne 'qemu-server.conf';
4484     }
4485
4486     my $storecfg = cfs_read_file('storage.cfg');
4487
4488     # destroy existing data - keep empty config
4489     my $vmcfgfn = PVE::QemuServer::config_file($vmid);
4490     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
4491
4492     my $tocmd = "/usr/lib/qemu-server/qmextract";
4493
4494     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
4495     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
4496     $tocmd .= ' --prealloc' if $opts->{prealloc};
4497     $tocmd .= ' --info' if $opts->{info};
4498
4499     # tar option "xf" does not autodetect compression when read from STDIN,
4500     # so we pipe to zcat
4501     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
4502         PVE::Tools::shellquote("--to-command=$tocmd");
4503
4504     my $tmpdir = "/var/tmp/vzdumptmp$$";
4505     mkpath $tmpdir;
4506
4507     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
4508     local $ENV{VZDUMP_VMID} = $vmid;
4509     local $ENV{VZDUMP_USER} = $user;
4510
4511     my $conffile = config_file($vmid);
4512     my $tmpfn = "$conffile.$$.tmp";
4513
4514     # disable interrupts (always do cleanups)
4515     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
4516         print STDERR "got interrupt - ignored\n";
4517     };
4518
4519     eval {
4520         # enable interrupts
4521         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
4522             die "interrupted by signal\n";
4523         };
4524
4525         if ($archive eq '-') {
4526             print "extracting archive from STDIN\n";
4527             run_command($cmd, input => "<&STDIN");
4528         } else {
4529             print "extracting archive '$archive'\n";
4530             run_command($cmd);
4531         }
4532
4533         return if $opts->{info};
4534
4535         # read new mapping
4536         my $map = {};
4537         my $statfile = "$tmpdir/qmrestore.stat";
4538         if (my $fd = IO::File->new($statfile, "r")) {
4539             while (defined (my $line = <$fd>)) {
4540                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4541                     $map->{$1} = $2 if $1;
4542                 } else {
4543                     print STDERR "unable to parse line in statfile - $line\n";
4544                 }
4545             }
4546             $fd->close();
4547         }
4548
4549         my $confsrc = "$tmpdir/qemu-server.conf";
4550
4551         my $srcfd = new IO::File($confsrc, "r") ||
4552             die "unable to open file '$confsrc'\n";
4553
4554         my $outfd = new IO::File ($tmpfn, "w") ||
4555             die "unable to write config for VM $vmid\n";
4556
4557         my $cookie = { netcount => 0 };
4558         while (defined (my $line = <$srcfd>)) {
4559             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
4560         }
4561
4562         $srcfd->close();
4563         $outfd->close();
4564     };
4565     my $err = $@;
4566
4567     if ($err) {
4568
4569         unlink $tmpfn;
4570
4571         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
4572
4573         die $err;
4574     }
4575
4576     rmtree $tmpdir;
4577
4578     rename $tmpfn, $conffile ||
4579         die "unable to commit configuration file '$conffile'\n";
4580
4581     PVE::Cluster::cfs_update(); # make sure we read new file
4582
4583     eval { rescan($vmid, 1); };
4584     warn $@ if $@;
4585 };
4586
4587
4588 # Internal snapshots
4589
4590 # NOTE: Snapshot create/delete involves several non-atomic
4591 # action, and can take a long time.
4592 # So we try to avoid locking the file and use 'lock' variable
4593 # inside the config file instead.
4594
4595 my $snapshot_copy_config = sub {
4596     my ($source, $dest) = @_;
4597
4598     foreach my $k (keys %$source) {
4599         next if $k eq 'snapshots';
4600         next if $k eq 'snapstate';
4601         next if $k eq 'snaptime';
4602         next if $k eq 'vmstate';
4603         next if $k eq 'lock';
4604         next if $k eq 'digest';
4605         next if $k eq 'description';
4606         next if $k =~ m/^unused\d+$/;
4607
4608         $dest->{$k} = $source->{$k};
4609     }
4610 };
4611
4612 my $snapshot_apply_config = sub {
4613     my ($conf, $snap) = @_;
4614
4615     # copy snapshot list
4616     my $newconf = {
4617         snapshots => $conf->{snapshots},
4618     };
4619
4620     # keep description and list of unused disks
4621     foreach my $k (keys %$conf) {
4622         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
4623         $newconf->{$k} = $conf->{$k};
4624     }
4625
4626     &$snapshot_copy_config($snap, $newconf);
4627
4628     return $newconf;
4629 };
4630
4631 sub foreach_writable_storage {
4632     my ($conf, $func) = @_;
4633
4634     my $sidhash = {};
4635
4636     foreach my $ds (keys %$conf) {
4637         next if !valid_drivename($ds);
4638
4639         my $drive = parse_drive($ds, $conf->{$ds});
4640         next if !$drive;
4641         next if drive_is_cdrom($drive);
4642
4643         my $volid = $drive->{file};
4644
4645         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4646         $sidhash->{$sid} = $sid if $sid;
4647     }
4648
4649     foreach my $sid (sort keys %$sidhash) {
4650         &$func($sid);
4651     }
4652 }
4653
4654 my $alloc_vmstate_volid = sub {
4655     my ($storecfg, $vmid, $conf, $snapname) = @_;
4656
4657     # Note: we try to be smart when selecting a $target storage
4658
4659     my $target;
4660
4661     # search shared storage first
4662     foreach_writable_storage($conf, sub {
4663         my ($sid) = @_;
4664         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
4665         return if !$scfg->{shared};
4666
4667         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
4668     });
4669
4670     if (!$target) {
4671         # now search local storage
4672         foreach_writable_storage($conf, sub {
4673             my ($sid) = @_;
4674             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
4675             return if $scfg->{shared};
4676
4677             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
4678         });
4679     }
4680
4681     $target = 'local' if !$target;
4682
4683     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
4684     # we abort live save after $conf->{memory}, so we need at max twice that space
4685     my $size = $conf->{memory}*2 + $driver_state_size;
4686
4687     my $name = "vm-$vmid-state-$snapname";
4688     my $scfg = PVE::Storage::storage_config($storecfg, $target);
4689     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
4690     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
4691
4692     return $volid;
4693 };
4694
4695 my $snapshot_prepare = sub {
4696     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
4697
4698     my $snap;
4699
4700     my $updatefn =  sub {
4701
4702         my $conf = load_config($vmid);
4703
4704         die "you can't take a snapshot if it's a template\n"
4705             if is_template($conf);
4706
4707         check_lock($conf);
4708
4709         $conf->{lock} = 'snapshot';
4710
4711         die "snapshot name '$snapname' already used\n"
4712             if defined($conf->{snapshots}->{$snapname});
4713
4714         my $storecfg = PVE::Storage::config();
4715         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
4716
4717         $snap = $conf->{snapshots}->{$snapname} = {};
4718
4719         if ($save_vmstate && check_running($vmid)) {
4720             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
4721         }
4722
4723         &$snapshot_copy_config($conf, $snap);
4724
4725         $snap->{snapstate} = "prepare";
4726         $snap->{snaptime} = time();
4727         $snap->{description} = $comment if $comment;
4728
4729         # always overwrite machine if we save vmstate. This makes sure we
4730         # can restore it later using correct machine type
4731         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
4732
4733         update_config_nolock($vmid, $conf, 1);
4734     };
4735
4736     lock_config($vmid, $updatefn);
4737
4738     return $snap;
4739 };
4740
4741 my $snapshot_commit = sub {
4742     my ($vmid, $snapname) = @_;
4743
4744     my $updatefn = sub {
4745
4746         my $conf = load_config($vmid);
4747
4748         die "missing snapshot lock\n"
4749             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
4750
4751         my $has_machine_config = defined($conf->{machine});
4752
4753         my $snap = $conf->{snapshots}->{$snapname};
4754
4755         die "snapshot '$snapname' does not exist\n" if !defined($snap);
4756
4757         die "wrong snapshot state\n"
4758             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
4759
4760         delete $snap->{snapstate};
4761         delete $conf->{lock};
4762
4763         my $newconf = &$snapshot_apply_config($conf, $snap);
4764
4765         delete $newconf->{machine} if !$has_machine_config;
4766
4767         $newconf->{parent} = $snapname;
4768
4769         update_config_nolock($vmid, $newconf, 1);
4770     };
4771
4772     lock_config($vmid, $updatefn);
4773 };
4774
4775 sub snapshot_rollback {
4776     my ($vmid, $snapname) = @_;
4777
4778     my $snap;
4779
4780     my $prepare = 1;
4781
4782     my $storecfg = PVE::Storage::config();
4783
4784     my $updatefn = sub {
4785
4786         my $conf = load_config($vmid);
4787
4788         die "you can't rollback if vm is a template\n" if is_template($conf);
4789
4790         $snap = $conf->{snapshots}->{$snapname};
4791
4792         die "snapshot '$snapname' does not exist\n" if !defined($snap);
4793
4794         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
4795             if $snap->{snapstate};
4796
4797         if ($prepare) {
4798             check_lock($conf);
4799             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
4800         }
4801
4802         die "unable to rollback vm $vmid: vm is running\n"
4803             if check_running($vmid);
4804
4805         if ($prepare) {
4806             $conf->{lock} = 'rollback';
4807         } else {
4808             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
4809             delete $conf->{lock};
4810         }
4811
4812         my $forcemachine;
4813
4814         if (!$prepare) {
4815             my $has_machine_config = defined($conf->{machine});
4816
4817             # copy snapshot config to current config
4818             $conf = &$snapshot_apply_config($conf, $snap);
4819             $conf->{parent} = $snapname;
4820
4821             # Note: old code did not store 'machine', so we try to be smart
4822             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
4823             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
4824             # we remove the 'machine' configuration if not explicitly specified
4825             # in the original config.
4826             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
4827         }
4828
4829         update_config_nolock($vmid, $conf, 1);
4830
4831         if (!$prepare && $snap->{vmstate}) {
4832             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
4833             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
4834         }
4835     };
4836
4837     lock_config($vmid, $updatefn);
4838
4839     foreach_drive($snap, sub {
4840         my ($ds, $drive) = @_;
4841
4842         return if drive_is_cdrom($drive);
4843
4844         my $volid = $drive->{file};
4845         my $device = "drive-$ds";
4846
4847         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
4848     });
4849
4850     $prepare = 0;
4851     lock_config($vmid, $updatefn);
4852 }
4853
4854 my $savevm_wait = sub {
4855     my ($vmid) = @_;
4856
4857     for(;;) {
4858         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
4859         if (!$stat->{status}) {
4860             die "savevm not active\n";
4861         } elsif ($stat->{status} eq 'active') {
4862             sleep(1);
4863             next;
4864         } elsif ($stat->{status} eq 'completed') {
4865             last;
4866         } else {
4867             die "query-savevm returned status '$stat->{status}'\n";
4868         }
4869     }
4870 };
4871
4872 sub snapshot_create {
4873     my ($vmid, $snapname, $save_vmstate, $freezefs, $comment) = @_;
4874
4875     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
4876
4877     $freezefs = $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
4878
4879     my $drivehash = {};
4880
4881     my $running = check_running($vmid);
4882
4883     eval {
4884         # create internal snapshots of all drives
4885
4886         my $storecfg = PVE::Storage::config();
4887
4888         if ($running) {
4889             if ($snap->{vmstate}) {
4890                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
4891                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
4892                 &$savevm_wait($vmid);
4893             } else {
4894                 vm_mon_cmd($vmid, "savevm-start");
4895             }
4896         };
4897
4898         qga_freezefs($vmid) if $running && $freezefs;
4899
4900         foreach_drive($snap, sub {
4901             my ($ds, $drive) = @_;
4902
4903             return if drive_is_cdrom($drive);
4904
4905             my $volid = $drive->{file};
4906             my $device = "drive-$ds";
4907
4908             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
4909             $drivehash->{$ds} = 1;
4910        });
4911     };
4912     my $err = $@;
4913
4914     eval { qga_unfreezefs($vmid) if $running && $freezefs; };
4915     warn $@ if $@;
4916
4917     eval { vm_mon_cmd($vmid, "savevm-end") if $running; };
4918     warn $@ if $@;
4919
4920     if ($err) {
4921         warn "snapshot create failed: starting cleanup\n";
4922         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
4923         warn $@ if $@;
4924         die $err;
4925     }
4926
4927     &$snapshot_commit($vmid, $snapname);
4928 }
4929
4930 # Note: $drivehash is only set when called from snapshot_create.
4931 sub snapshot_delete {
4932     my ($vmid, $snapname, $force, $drivehash) = @_;
4933
4934     my $prepare = 1;
4935
4936     my $snap;
4937     my $unused = [];
4938
4939     my $unlink_parent = sub {
4940         my ($confref, $new_parent) = @_;
4941
4942         if ($confref->{parent} && $confref->{parent} eq $snapname) {
4943             if ($new_parent) {
4944                 $confref->{parent} = $new_parent;
4945             } else {
4946                 delete $confref->{parent};
4947             }
4948         }
4949     };
4950
4951     my $updatefn =  sub {
4952         my ($remove_drive) = @_;
4953
4954         my $conf = load_config($vmid);
4955
4956         if (!$drivehash) {
4957             check_lock($conf);
4958             die "you can't delete a snapshot if vm is a template\n"
4959                 if is_template($conf);
4960         }
4961
4962         $snap = $conf->{snapshots}->{$snapname};
4963
4964         die "snapshot '$snapname' does not exist\n" if !defined($snap);
4965
4966         # remove parent refs
4967         if (!$prepare) {
4968             &$unlink_parent($conf, $snap->{parent});
4969             foreach my $sn (keys %{$conf->{snapshots}}) {
4970                 next if $sn eq $snapname;
4971                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
4972             }
4973         }
4974
4975         if ($remove_drive) {
4976             if ($remove_drive eq 'vmstate') {
4977                 delete $snap->{$remove_drive};
4978             } else {
4979                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
4980                 my $volid = $drive->{file};
4981                 delete $snap->{$remove_drive};
4982                 add_unused_volume($conf, $volid);
4983             }
4984         }
4985
4986         if ($prepare) {
4987             $snap->{snapstate} = 'delete';
4988         } else {
4989             delete $conf->{snapshots}->{$snapname};
4990             delete $conf->{lock} if $drivehash;
4991             foreach my $volid (@$unused) {
4992                 add_unused_volume($conf, $volid);
4993             }
4994         }
4995
4996         update_config_nolock($vmid, $conf, 1);
4997     };
4998
4999     lock_config($vmid, $updatefn);
5000
5001     # now remove vmstate file
5002
5003     my $storecfg = PVE::Storage::config();
5004
5005     if ($snap->{vmstate}) {
5006         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5007         if (my $err = $@) {
5008             die $err if !$force;
5009             warn $err;
5010         }
5011         # save changes (remove vmstate from snapshot)
5012         lock_config($vmid, $updatefn, 'vmstate') if !$force;
5013     };
5014
5015     # now remove all internal snapshots
5016     foreach_drive($snap, sub {
5017         my ($ds, $drive) = @_;
5018
5019         return if drive_is_cdrom($drive);
5020
5021         my $volid = $drive->{file};
5022         my $device = "drive-$ds";
5023
5024         if (!$drivehash || $drivehash->{$ds}) {
5025             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
5026             if (my $err = $@) {
5027                 die $err if !$force;
5028                 warn $err;
5029             }
5030         }
5031
5032         # save changes (remove drive fron snapshot)
5033         lock_config($vmid, $updatefn, $ds) if !$force;
5034         push @$unused, $volid;
5035     });
5036
5037     # now cleanup config
5038     $prepare = 0;
5039     lock_config($vmid, $updatefn);
5040 }
5041
5042 sub has_feature {
5043     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
5044
5045     my $err;
5046     foreach_drive($conf, sub {
5047         my ($ds, $drive) = @_;
5048
5049         return if drive_is_cdrom($drive);
5050         my $volid = $drive->{file};
5051         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
5052     });
5053
5054     return $err ? 0 : 1;
5055 }
5056
5057 sub template_create {
5058     my ($vmid, $conf, $disk) = @_;
5059
5060     my $storecfg = PVE::Storage::config();
5061
5062     foreach_drive($conf, sub {
5063         my ($ds, $drive) = @_;
5064
5065         return if drive_is_cdrom($drive);
5066         return if $disk && $ds ne $disk;
5067
5068         my $volid = $drive->{file};
5069         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5070
5071         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5072         $drive->{file} = $voliddst;
5073         $conf->{$ds} = print_drive($vmid, $drive);
5074         update_config_nolock($vmid, $conf, 1);
5075     });
5076 }
5077
5078 sub is_template {
5079     my ($conf) = @_;
5080
5081     return 1 if defined $conf->{template} && $conf->{template} == 1;
5082 }
5083
5084 sub qemu_img_convert {
5085     my ($src_volid, $dst_volid, $size, $snapname) = @_;
5086
5087     my $storecfg = PVE::Storage::config();
5088     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5089     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5090
5091     if ($src_storeid && $dst_storeid) {
5092         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5093         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5094
5095         my $src_format = qemu_img_format($src_scfg, $src_volname);
5096         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5097
5098         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5099         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5100
5101         my $cmd = [];
5102         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5103         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5104         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
5105
5106         my $parser = sub {
5107             my $line = shift;
5108             if($line =~ m/\((\S+)\/100\%\)/){
5109                 my $percent = $1;
5110                 my $transferred = int($size * $percent / 100);
5111                 my $remaining = $size - $transferred;
5112
5113                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5114             }
5115
5116         };
5117
5118         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5119         my $err = $@;
5120         die "copy failed: $err" if $err;
5121     }
5122 }
5123
5124 sub qemu_img_format {
5125     my ($scfg, $volname) = @_;
5126
5127     if ($scfg->{path} && $volname =~ m/\.(raw|qcow2|qed|vmdk)$/) {
5128         return $1;
5129     } elsif ($scfg->{type} eq 'iscsi') {
5130         return "host_device";
5131     } else {
5132         return "raw";
5133     }
5134 }
5135
5136 sub qemu_drive_mirror {
5137     my ($vmid, $drive, $dst_volid, $vmiddst, $maxwait) = @_;
5138
5139     my $count = 1;
5140     my $old_len = 0;
5141     my $frozen = undef;
5142
5143     my $storecfg = PVE::Storage::config();
5144     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5145
5146     if ($dst_storeid) {
5147         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5148
5149         my $format;
5150         if ($dst_volname =~ m/\.(raw|qcow2)$/){
5151             $format = $1;
5152         }
5153
5154         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5155
5156         if ($format) {
5157             #fixme : sometime drive-mirror timeout, but works fine after.
5158             # (I have see the problem with big volume > 200GB), so we need to eval
5159             eval { vm_mon_cmd($vmid, "drive-mirror", timeout => 10, device => "drive-$drive", mode => "existing",
5160                               sync => "full", target => $dst_path, format => $format); };
5161         } else {
5162             eval { vm_mon_cmd($vmid, "drive-mirror", timeout => 10, device => "drive-$drive", mode => "existing",
5163                               sync => "full", target => $dst_path); };
5164         }
5165
5166         eval {
5167             while (1) {
5168                 my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5169                 my $stat = @$stats[0];
5170                 die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5171                 die "error job is not mirroring" if $stat->{type} ne "mirror";
5172
5173                 my $transferred = $stat->{offset};
5174                 my $total = $stat->{len};
5175                 my $remaining = $total - $transferred;
5176                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5177
5178                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent %\n";
5179
5180                 last if ($stat->{len} == $stat->{offset});
5181                 if ($old_len == $stat->{offset}) {
5182                     if ($maxwait && $count > $maxwait) {
5183                     # if writes to disk occurs the disk needs to be freezed
5184                     # to be able to complete the migration
5185                         vm_suspend($vmid,1);
5186                         $count = 0;
5187                         $frozen = 1;
5188                     } else {
5189                         $count++ unless $frozen;
5190                     }
5191                 } elsif ($frozen) {
5192                     vm_resume($vmid,1);
5193                     $count = 0;
5194                 }
5195                 $old_len = $stat->{offset};
5196                 sleep 1;
5197             }
5198
5199             if ($vmiddst == $vmid) {
5200                 # switch the disk if source and destination are on the same guest
5201                 vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive");
5202             }
5203         };
5204         if (my $err = $@) {
5205             eval { vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive"); };
5206             die "mirroring error: $err";
5207         }
5208
5209         if ($vmiddst != $vmid) {
5210             # if we clone a disk for a new target vm, we don't switch the disk
5211             vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5212         }
5213     }
5214 }
5215
5216 sub clone_disk {
5217     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5218         $newvmid, $storage, $format, $full, $newvollist) = @_;
5219
5220     my $newvolid;
5221
5222     if (!$full) {
5223         print "create linked clone of drive $drivename ($drive->{file})\n";
5224         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5225         push @$newvollist, $newvolid;
5226     } else {
5227         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5228         $storeid = $storage if $storage;
5229
5230         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5231         if (!$format) {
5232             $format = $drive->{format} || $defFormat;
5233         }
5234
5235         # test if requested format is supported - else use default
5236         my $supported = grep { $_ eq $format } @$validFormats;
5237         $format = $defFormat if !$supported;
5238
5239         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5240
5241         print "create full clone of drive $drivename ($drive->{file})\n";
5242         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5243         push @$newvollist, $newvolid;
5244
5245         if (!$running || $snapname) {
5246             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
5247         } else {
5248             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
5249         }
5250     }
5251
5252     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5253
5254     my $disk = $drive;
5255     $disk->{format} = undef;
5256     $disk->{file} = $newvolid;
5257     $disk->{size} = $size;
5258
5259     return $disk;
5260 }
5261
5262 # this only works if VM is running
5263 sub get_current_qemu_machine {
5264     my ($vmid) = @_;
5265
5266     my $cmd = { execute => 'query-machines', arguments => {} };
5267     my $res = PVE::QemuServer::vm_qmp_command($vmid, $cmd);
5268
5269     my ($current, $default);
5270     foreach my $e (@$res) {
5271         $default = $e->{name} if $e->{'is-default'};
5272         $current = $e->{name} if $e->{'is-current'};
5273     }
5274
5275     # fallback to the default machine if current is not supported by qemu
5276     return $current || $default || 'pc';
5277 }
5278
5279 sub lspci {
5280
5281     my $devices = {};
5282
5283     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
5284             my (undef, $id, $function) = @_;
5285             my $res = { id => $id, function => $function};
5286             push @{$devices->{$id}}, $res;
5287     });
5288
5289     return $devices;
5290 }
5291
5292 1;