PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 unless(defined(&_VZSYSCALLS_H_)) {
  71     eval 'sub _VZSYSCALLS_H_ () {1;}' unless defined(&_VZSYSCALLS_H_);
  72     require 'sys/syscall.ph';
  73     if(defined(&__x86_64__)) {
  74         eval 'sub __NR_fairsched_vcpus () {499;}' unless defined(&__NR_fairsched_vcpus);
  75         eval 'sub __NR_fairsched_mknod () {504;}' unless defined(&__NR_fairsched_mknod);
  76         eval 'sub __NR_fairsched_rmnod () {505;}' unless defined(&__NR_fairsched_rmnod);
  77         eval 'sub __NR_fairsched_chwt () {506;}' unless defined(&__NR_fairsched_chwt);
  78         eval 'sub __NR_fairsched_mvpr () {507;}' unless defined(&__NR_fairsched_mvpr);
  79         eval 'sub __NR_fairsched_rate () {508;}' unless defined(&__NR_fairsched_rate);
  80         eval 'sub __NR_setluid () {501;}' unless defined(&__NR_setluid);
  81         eval 'sub __NR_setublimit () {502;}' unless defined(&__NR_setublimit);
  82     }
  83     elsif(defined( &__i386__) ) {
  84         eval 'sub __NR_fairsched_mknod () {500;}' unless defined(&__NR_fairsched_mknod);
  85         eval 'sub __NR_fairsched_rmnod () {501;}' unless defined(&__NR_fairsched_rmnod);
  86         eval 'sub __NR_fairsched_chwt () {502;}' unless defined(&__NR_fairsched_chwt);
  87         eval 'sub __NR_fairsched_mvpr () {503;}' unless defined(&__NR_fairsched_mvpr);
  88         eval 'sub __NR_fairsched_rate () {504;}' unless defined(&__NR_fairsched_rate);
  89         eval 'sub __NR_fairsched_vcpus () {505;}' unless defined(&__NR_fairsched_vcpus);
  90         eval 'sub __NR_setluid () {511;}' unless defined(&__NR_setluid);
  91         eval 'sub __NR_setublimit () {512;}' unless defined(&__NR_setublimit);
  92     } else {
  93         die("no fairsched syscall for this arch");
  94     }
  95     require 'asm/ioctl.ph';
  96     eval 'sub KVM_GET_API_VERSION () { &_IO(0xAE, 0x);}' unless defined(&KVM_GET_API_VERSION);
  97 }
  98
  99 sub fairsched_mknod {
 100     my ($parent, $weight, $desired) = @_;
 101
 102     return syscall(&__NR_fairsched_mknod, int($parent), int($weight), int($desired));
 103 }
 104
 105 sub fairsched_rmnod {
 106     my ($id) = @_;
 107
 108     return syscall(&__NR_fairsched_rmnod, int($id));
 109 }
 110
 111 sub fairsched_mvpr {
 112     my ($pid, $newid) = @_;
 113
 114     return syscall(&__NR_fairsched_mvpr, int($pid), int($newid));
 115 }
 116
 117 sub fairsched_vcpus {
 118     my ($id, $vcpus) = @_;
 119
 120     return syscall(&__NR_fairsched_vcpus, int($id), int($vcpus));
 121 }
 122
 123 sub fairsched_rate {
 124     my ($id, $op, $rate) = @_;
 125
 126     return syscall(&__NR_fairsched_rate, int($id), int($op), int($rate));
 127 }
 128
 129 use constant FAIRSCHED_SET_RATE  => 0;
 130 use constant FAIRSCHED_DROP_RATE => 1;
 131 use constant FAIRSCHED_GET_RATE  => 2;
 132
 133 sub fairsched_cpulimit {
 134     my ($id, $limit) = @_;
 135
 136     my $cpulim1024 = int($limit * 1024 / 100);
 137     my $op = $cpulim1024 ? FAIRSCHED_SET_RATE : FAIRSCHED_DROP_RATE;
 138
 139     return fairsched_rate($id, $op, $cpulim1024);
 140 }
 141
 142 my $nodename = PVE::INotify::nodename();
 143
 144 mkdir "/etc/pve/nodes/$nodename";
 145 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 146 mkdir $confdir;
 147
 148 my $var_run_tmpdir = "/var/run/qemu-server";
 149 mkdir $var_run_tmpdir;
 150
 151 my $lock_dir = "/var/lock/qemu-server";
 152 mkdir $lock_dir;
 153
 154 my $pcisysfs = "/sys/bus/pci";
 155
 156 my $confdesc = {
 157     onboot => {
 158         optional => 1,
 159         type => 'boolean',
 160         description => "Specifies whether a VM will be started during system bootup.",
 161         default => 0,
 162     },
 163     autostart => {
 164         optional => 1,
 165         type => 'boolean',
 166         description => "Automatic restart after crash (currently ignored).",
 167         default => 0,
 168     },
 169     hotplug => {
 170         optional => 1,
 171         type => 'string', format => 'pve-hotplug-features',
 172         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 173         default => 'network,disk,usb',
 174     },
 175     reboot => {
 176         optional => 1,
 177         type => 'boolean',
 178         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 179         default => 1,
 180     },
 181     lock => {
 182         optional => 1,
 183         type => 'string',
 184         description => "Lock/unlock the VM.",
 185         enum => [qw(migrate backup snapshot rollback)],
 186     },
 187     cpulimit => {
 188         optional => 1,
 189         type => 'integer',
 190         description => "Limit of CPU usage in per cent. Note if the computer has 2 CPUs, it has total of 200% CPU time. Value '0' indicates no CPU limit.\n\nNOTE: This option is currently ignored.",
 191         minimum => 0,
 192         default => 0,
 193     },
 194     cpuunits => {
 195         optional => 1,
 196         type => 'integer',
 197         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 198         minimum => 0,
 199         maximum => 500000,
 200         default => 1000,
 201     },
 202     memory => {
 203         optional => 1,
 204         type => 'integer',
 205         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 206         minimum => 16,
 207         default => 512,
 208     },
 209     balloon => {
 210         optional => 1,
 211         type => 'integer',
 212         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 213         minimum => 0,
 214     },
 215     shares => {
 216         optional => 1,
 217         type => 'integer',
 218         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 219         minimum => 0,
 220         maximum => 50000,
 221         default => 1000,
 222     },
 223     keyboard => {
 224         optional => 1,
 225         type => 'string',
 226         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 227         enum => PVE::Tools::kvmkeymaplist(),
 228         default => 'en-us',
 229     },
 230     name => {
 231         optional => 1,
 232         type => 'string', format => 'dns-name',
 233         description => "Set a name for the VM. Only used on the configuration web interface.",
 234     },
 235     scsihw => {
 236         optional => 1,
 237         type => 'string',
 238         description => "scsi controller model",
 239         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 240         default => 'lsi',
 241     },
 242     description => {
 243         optional => 1,
 244         type => 'string',
 245         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 246     },
 247     ostype => {
 248         optional => 1,
 249         type => 'string',
 250         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 251         description => <<EODESC,
 252 Used to enable special optimization/features for specific
 253 operating systems:
 254
 255 other  => unspecified OS
 256 wxp    => Microsoft Windows XP
 257 w2k    => Microsoft Windows 2000
 258 w2k3   => Microsoft Windows 2003
 259 w2k8   => Microsoft Windows 2008
 260 wvista => Microsoft Windows Vista
 261 win7   => Microsoft Windows 7
 262 win8   => Microsoft Windows 8/2012
 263 l24    => Linux 2.4 Kernel
 264 l26    => Linux 2.6/3.X Kernel
 265 solaris => solaris/opensolaris/openindiania kernel
 266
 267 other|l24|l26|solaris                       ... no special behaviour
 268 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 269 EODESC
 270     },
 271     boot => {
 272         optional => 1,
 273         type => 'string',
 274         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 275         pattern => '[acdn]{1,4}',
 276         default => 'cdn',
 277     },
 278     bootdisk => {
 279         optional => 1,
 280         type => 'string', format => 'pve-qm-bootdisk',
 281         description => "Enable booting from specified disk.",
 282         pattern => '(ide|sata|scsi|virtio)\d+',
 283     },
 284     smp => {
 285         optional => 1,
 286         type => 'integer',
 287         description => "The number of CPUs. Please use option -sockets instead.",
 288         minimum => 1,
 289         default => 1,
 290     },
 291     sockets => {
 292         optional => 1,
 293         type => 'integer',
 294         description => "The number of CPU sockets.",
 295         minimum => 1,
 296         default => 1,
 297     },
 298     cores => {
 299         optional => 1,
 300         type => 'integer',
 301         description => "The number of cores per socket.",
 302         minimum => 1,
 303         default => 1,
 304     },
 305     numa => {
 306         optional => 1,
 307         type => 'boolean',
 308         description => "Enable/disable Numa.",
 309         default => 0,
 310     },
 311     vcpus => {
 312         optional => 1,
 313         type => 'integer',
 314         description => "Number of hotplugged vcpus.",
 315         minimum => 1,
 316         default => 0,
 317     },
 318     acpi => {
 319         optional => 1,
 320         type => 'boolean',
 321         description => "Enable/disable ACPI.",
 322         default => 1,
 323     },
 324     agent => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Enable/disable Qemu GuestAgent.",
 328         default => 0,
 329     },
 330     kvm => {
 331         optional => 1,
 332         type => 'boolean',
 333         description => "Enable/disable KVM hardware virtualization.",
 334         default => 1,
 335     },
 336     tdf => {
 337         optional => 1,
 338         type => 'boolean',
 339         description => "Enable/disable time drift fix.",
 340         default => 0,
 341     },
 342     localtime => {
 343         optional => 1,
 344         type => 'boolean',
 345         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 346     },
 347     freeze => {
 348         optional => 1,
 349         type => 'boolean',
 350         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 351     },
 352     vga => {
 353         optional => 1,
 354         type => 'string',
 355         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 356         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 357     },
 358     watchdog => {
 359         optional => 1,
 360         type => 'string', format => 'pve-qm-watchdog',
 361         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 362         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 363     },
 364     startdate => {
 365         optional => 1,
 366         type => 'string',
 367         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 368         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 369         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 370         default => 'now',
 371     },
 372     startup =>  get_standard_option('pve-startup-order'),
 373     template => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Enable/disable Template.",
 377         default => 0,
 378     },
 379     args => {
 380         optional => 1,
 381         type => 'string',
 382         description => <<EODESCR,
 383 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 384
 385 args: -no-reboot -no-hpet
 386 EODESCR
 387     },
 388     tablet => {
 389         optional => 1,
 390         type => 'boolean',
 391         default => 1,
 392         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 393     },
 394     migrate_speed => {
 395         optional => 1,
 396         type => 'integer',
 397         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 398         minimum => 0,
 399         default => 0,
 400     },
 401     migrate_downtime => {
 402         optional => 1,
 403         type => 'number',
 404         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 405         minimum => 0,
 406         default => 0.1,
 407     },
 408     cdrom => {
 409         optional => 1,
 410         type => 'string', format => 'pve-qm-drive',
 411         typetext => 'volume',
 412         description => "This is an alias for option -ide2",
 413     },
 414     cpu => {
 415         optional => 1,
 416         description => "Emulated CPU type.",
 417         type => 'string',
 418         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 419         default => 'kvm64',
 420     },
 421     parent => get_standard_option('pve-snapshot-name', {
 422         optional => 1,
 423         description => "Parent snapshot name. This is used internally, and should not be modified.",
 424     }),
 425     snaptime => {
 426         optional => 1,
 427         description => "Timestamp for snapshots.",
 428         type => 'integer',
 429         minimum => 0,
 430     },
 431     vmstate => {
 432         optional => 1,
 433         type => 'string', format => 'pve-volume-id',
 434         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 435     },
 436     machine => {
 437         description => "Specific the Qemu machine type.",
 438         type => 'string',
 439         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 440         maxLength => 40,
 441         optional => 1,
 442     },
 443     smbios1 => {
 444         description => "Specify SMBIOS type 1 fields.",
 445         type => 'string', format => 'pve-qm-smbios1',
 446         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 447         maxLength => 256,
 448         optional => 1,
 449     },
 450 };
 451
 452 # what about other qemu settings ?
 453 #cpu => 'string',
 454 #machine => 'string',
 455 #fda => 'file',
 456 #fdb => 'file',
 457 #mtdblock => 'file',
 458 #sd => 'file',
 459 #pflash => 'file',
 460 #snapshot => 'bool',
 461 #bootp => 'file',
 462 ##tftp => 'dir',
 463 ##smb => 'dir',
 464 #kernel => 'file',
 465 #append => 'string',
 466 #initrd => 'file',
 467 ##soundhw => 'string',
 468
 469 while (my ($k, $v) = each %$confdesc) {
 470     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 471 }
 472
 473 my $MAX_IDE_DISKS = 4;
 474 my $MAX_SCSI_DISKS = 14;
 475 my $MAX_VIRTIO_DISKS = 16;
 476 my $MAX_SATA_DISKS = 6;
 477 my $MAX_USB_DEVICES = 5;
 478 my $MAX_NETS = 32;
 479 my $MAX_UNUSED_DISKS = 8;
 480 my $MAX_HOSTPCI_DEVICES = 4;
 481 my $MAX_SERIAL_PORTS = 4;
 482 my $MAX_PARALLEL_PORTS = 3;
 483 my $MAX_NUMA = 8;
 484 my $MAX_MEM = 4194304;
 485 my $STATICMEM = 1024;
 486
 487 my $numadesc = {
 488     optional => 1,
 489     type => 'string', format => 'pve-qm-numanode',
 490     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 491     description => "numa topology",
 492 };
 493 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 494
 495 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 496     $confdesc->{"numa$i"} = $numadesc;
 497 }
 498
 499 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 500                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 501                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 502 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 503
 504 my $netdesc = {
 505     optional => 1,
 506     type => 'string', format => 'pve-qm-net',
 507     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 508     description => <<EODESCR,
 509 Specify network devices.
 510
 511 MODEL is one of: $nic_model_list_txt
 512
 513 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 514 automatically generated if not specified.
 515
 516 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 517
 518 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 519
 520 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 521
 522 10.0.2.2   Gateway
 523 10.0.2.3   DNS Server
 524 10.0.2.4   SMB Server
 525
 526 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 527
 528 EODESCR
 529 };
 530 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 531
 532 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 533     $confdesc->{"net$i"} = $netdesc;
 534 }
 535
 536 my $drivename_hash;
 537
 538 my $idedesc = {
 539     optional => 1,
 540     type => 'string', format => 'pve-qm-drive',
 541     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 542     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 543 };
 544 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 545
 546 my $scsidesc = {
 547     optional => 1,
 548     type => 'string', format => 'pve-qm-drive',
 549     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>]',
 550     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 551 };
 552 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 553
 554 my $satadesc = {
 555     optional => 1,
 556     type => 'string', format => 'pve-qm-drive',
 557     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 558     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 559 };
 560 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 561
 562 my $virtiodesc = {
 563     optional => 1,
 564     type => 'string', format => 'pve-qm-drive',
 565     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
 566     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 567 };
 568 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 569
 570 my $usbdesc = {
 571     optional => 1,
 572     type => 'string', format => 'pve-qm-usb-device',
 573     typetext => 'host=HOSTUSBDEVICE|spice',
 574     description => <<EODESCR,
 575 Configure an USB device (n is 0 to 4). This can be used to
 576 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 577
 578 'bus-port(.port)*' (decimal numbers) or
 579 'vendor_id:product_id' (hexadeciaml numbers)
 580
 581 You can use the 'lsusb -t' command to list existing usb devices.
 582
 583 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 584
 585 The value 'spice' can be used to add a usb redirection devices for spice.
 586
 587 EODESCR
 588 };
 589 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 590
 591 my $hostpcidesc = {
 592         optional => 1,
 593         type => 'string', format => 'pve-qm-hostpci',
 594         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 595         description => <<EODESCR,
 596 Map host pci devices. HOSTPCIDEVICE syntax is:
 597
 598 'bus:dev.func' (hexadecimal numbers)
 599
 600 You can us the 'lspci' command to list existing pci devices.
 601
 602 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 603
 604 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 605
 606 Experimental: user reported problems with this option.
 607 EODESCR
 608 };
 609 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 610
 611 my $serialdesc = {
 612         optional => 1,
 613         type => 'string',
 614         pattern => '(/dev/.+|socket)',
 615         description =>  <<EODESCR,
 616 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 617
 618 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 619
 620 Experimental: user reported problems with this option.
 621 EODESCR
 622 };
 623
 624 my $paralleldesc= {
 625         optional => 1,
 626         type => 'string',
 627         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 628         description =>  <<EODESCR,
 629 Map host parallel devices (n is 0 to 2).
 630
 631 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 632
 633 Experimental: user reported problems with this option.
 634 EODESCR
 635 };
 636
 637 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 638     $confdesc->{"parallel$i"} = $paralleldesc;
 639 }
 640
 641 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 642     $confdesc->{"serial$i"} = $serialdesc;
 643 }
 644
 645 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 646     $confdesc->{"hostpci$i"} = $hostpcidesc;
 647 }
 648
 649 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 650     $drivename_hash->{"ide$i"} = 1;
 651     $confdesc->{"ide$i"} = $idedesc;
 652 }
 653
 654 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 655     $drivename_hash->{"sata$i"} = 1;
 656     $confdesc->{"sata$i"} = $satadesc;
 657 }
 658
 659 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 660     $drivename_hash->{"scsi$i"} = 1;
 661     $confdesc->{"scsi$i"} = $scsidesc ;
 662 }
 663
 664 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 665     $drivename_hash->{"virtio$i"} = 1;
 666     $confdesc->{"virtio$i"} = $virtiodesc;
 667 }
 668
 669 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 670     $confdesc->{"usb$i"} = $usbdesc;
 671 }
 672
 673 my $unuseddesc = {
 674     optional => 1,
 675     type => 'string', format => 'pve-volume-id',
 676     description => "Reference to unused volumes.",
 677 };
 678
 679 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 680     $confdesc->{"unused$i"} = $unuseddesc;
 681 }
 682
 683 my $kvm_api_version = 0;
 684
 685 sub kvm_version {
 686
 687     return $kvm_api_version if $kvm_api_version;
 688
 689     my $fh = IO::File->new("</dev/kvm") ||
 690         return 0;
 691
 692     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 693         $kvm_api_version = $v;
 694     }
 695
 696     $fh->close();
 697
 698     return  $kvm_api_version;
 699 }
 700
 701 my $kvm_user_version;
 702
 703 sub kvm_user_version {
 704
 705     return $kvm_user_version if $kvm_user_version;
 706
 707     $kvm_user_version = 'unknown';
 708
 709     my $tmp = `kvm -help 2>/dev/null`;
 710
 711     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 712         $kvm_user_version = $2;
 713     }
 714
 715     return $kvm_user_version;
 716
 717 }
 718
 719 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 720
 721 sub disknames {
 722     # order is important - used to autoselect boot disk
 723     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 724             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 725             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 726             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 727 }
 728
 729 sub valid_drivename {
 730     my $dev = shift;
 731
 732     return defined($drivename_hash->{$dev});
 733 }
 734
 735 sub option_exists {
 736     my $key = shift;
 737     return defined($confdesc->{$key});
 738 }
 739
 740 sub nic_models {
 741     return $nic_model_list;
 742 }
 743
 744 sub os_list_description {
 745
 746     return {
 747         other => 'Other',
 748         wxp => 'Windows XP',
 749         w2k => 'Windows 2000',
 750         w2k3 =>, 'Windows 2003',
 751         w2k8 => 'Windows 2008',
 752         wvista => 'Windows Vista',
 753         win7 => 'Windows 7',
 754         win8 => 'Windows 8/2012',
 755         l24 => 'Linux 2.4',
 756         l26 => 'Linux 2.6',
 757     };
 758 }
 759
 760 my $cdrom_path;
 761
 762 sub get_cdrom_path {
 763
 764     return  $cdrom_path if $cdrom_path;
 765
 766     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 767     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 768     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 769 }
 770
 771 sub get_iso_path {
 772     my ($storecfg, $vmid, $cdrom) = @_;
 773
 774     if ($cdrom eq 'cdrom') {
 775         return get_cdrom_path();
 776     } elsif ($cdrom eq 'none') {
 777         return '';
 778     } elsif ($cdrom =~ m|^/|) {
 779         return $cdrom;
 780     } else {
 781         return PVE::Storage::path($storecfg, $cdrom);
 782     }
 783 }
 784
 785 # try to convert old style file names to volume IDs
 786 sub filename_to_volume_id {
 787     my ($vmid, $file, $media) = @_;
 788
 789     if (!($file eq 'none' || $file eq 'cdrom' ||
 790           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 791
 792         return undef if $file =~ m|/|;
 793
 794         if ($media && $media eq 'cdrom') {
 795             $file = "local:iso/$file";
 796         } else {
 797             $file = "local:$vmid/$file";
 798         }
 799     }
 800
 801     return $file;
 802 }
 803
 804 sub verify_media_type {
 805     my ($opt, $vtype, $media) = @_;
 806
 807     return if !$media;
 808
 809     my $etype;
 810     if ($media eq 'disk') {
 811         $etype = 'images';
 812     } elsif ($media eq 'cdrom') {
 813         $etype = 'iso';
 814     } else {
 815         die "internal error";
 816     }
 817
 818     return if ($vtype eq $etype);
 819
 820     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 821 }
 822
 823 sub cleanup_drive_path {
 824     my ($opt, $storecfg, $drive) = @_;
 825
 826     # try to convert filesystem paths to volume IDs
 827
 828     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 829         ($drive->{file} !~ m|^/dev/.+|) &&
 830         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 831         ($drive->{file} !~ m/^\d+$/)) {
 832         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 833         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 834         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 835         verify_media_type($opt, $vtype, $drive->{media});
 836         $drive->{file} = $volid;
 837     }
 838
 839     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 840 }
 841
 842 sub create_conf_nolock {
 843     my ($vmid, $settings) = @_;
 844
 845     my $filename = config_file($vmid);
 846
 847     die "configuration file '$filename' already exists\n" if -f $filename;
 848
 849     my $defaults = load_defaults();
 850
 851     $settings->{name} = "vm$vmid" if !$settings->{name};
 852     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 853
 854     my $data = '';
 855     foreach my $opt (keys %$settings) {
 856         next if !$confdesc->{$opt};
 857
 858         my $value = $settings->{$opt};
 859         next if !$value;
 860
 861         $data .= "$opt: $value\n";
 862     }
 863
 864     PVE::Tools::file_set_contents($filename, $data);
 865 }
 866
 867 sub parse_hotplug_features {
 868     my ($data) = @_;
 869
 870     my $res = {};
 871
 872     return $res if $data eq '0';
 873
 874     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 875
 876     foreach my $feature (PVE::Tools::split_list($data)) {
 877         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 878             $res->{$1} = 1;
 879         } else {
 880             warn "ignoring unknown hotplug feature '$feature'\n";
 881         }
 882     }
 883     return $res;
 884 }
 885
 886 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 887 sub pve_verify_hotplug_features {
 888     my ($value, $noerr) = @_;
 889
 890     return $value if parse_hotplug_features($value);
 891
 892     return undef if $noerr;
 893
 894     die "unable to parse hotplug option\n";
 895 }
 896
 897 my $parse_size = sub {
 898     my ($value) = @_;
 899
 900     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 901     my ($size, $unit) = ($1, $3);
 902     if ($unit) {
 903         if ($unit eq 'K') {
 904             $size = $size * 1024;
 905         } elsif ($unit eq 'M') {
 906             $size = $size * 1024 * 1024;
 907         } elsif ($unit eq 'G') {
 908             $size = $size * 1024 * 1024 * 1024;
 909         }
 910     }
 911     return int($size);
 912 };
 913
 914 my $format_size = sub {
 915     my ($size) = @_;
 916
 917     $size = int($size);
 918
 919     my $kb = int($size/1024);
 920     return $size if $kb*1024 != $size;
 921
 922     my $mb = int($kb/1024);
 923     return "${kb}K" if $mb*1024 != $kb;
 924
 925     my $gb = int($mb/1024);
 926     return "${mb}M" if $gb*1024 != $mb;
 927
 928     return "${gb}G";
 929 };
 930
 931 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 932 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 933 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 934 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 935
 936 sub parse_drive {
 937     my ($key, $data) = @_;
 938
 939     my $res = {};
 940
 941     # $key may be undefined - used to verify JSON parameters
 942     if (!defined($key)) {
 943         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 944         $res->{index} = 0;
 945     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 946         $res->{interface} = $1;
 947         $res->{index} = $2;
 948     } else {
 949         return undef;
 950     }
 951
 952     foreach my $p (split (/,/, $data)) {
 953         next if $p =~ m/^\s*$/;
 954
 955         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues)=(.+)$/) {
 956             my ($k, $v) = ($1, $2);
 957
 958             $k = 'file' if $k eq 'volume';
 959
 960             return undef if defined $res->{$k};
 961
 962             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 963                 return undef if !$v || $v !~ m/^\d+/;
 964                 $k = "m$k";
 965                 $v = sprintf("%.3f", $v / (1024*1024));
 966             }
 967             $res->{$k} = $v;
 968         } else {
 969             if (!$res->{file} && $p !~ m/=/) {
 970                 $res->{file} = $p;
 971             } else {
 972                 return undef;
 973             }
 974         }
 975     }
 976
 977     return undef if !$res->{file};
 978
 979     if($res->{file} =~ m/\.(raw|cow|qcow|qcow2|vmdk|cloop)$/){
 980         $res->{format} = $1;
 981     }
 982
 983     return undef if $res->{cache} &&
 984         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 985     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 986     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 987     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 988     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 989     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 990     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 991     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qcow2|vmdk|cloop)$/;
 992     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 993     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 994     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 995     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 996     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 997     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 998     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 999
1000     return undef if $res->{mbps_rd} && $res->{mbps};
1001     return undef if $res->{mbps_wr} && $res->{mbps};
1002
1003     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
1004     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
1005     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
1006     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
1007     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
1008     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
1009
1010     return undef if $res->{iops_rd} && $res->{iops};
1011     return undef if $res->{iops_wr} && $res->{iops};
1012
1013
1014     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
1015     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
1016     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
1017     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
1018     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
1019     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
1020
1021
1022     if ($res->{size}) {
1023         return undef if !defined($res->{size} = &$parse_size($res->{size}));
1024     }
1025
1026     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1027         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1028         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1029         return undef if $res->{interface} eq 'virtio';
1030     }
1031
1032     # rerror does not work with scsi drives
1033     if ($res->{rerror}) {
1034         return undef if $res->{interface} eq 'scsi';
1035     }
1036
1037     return $res;
1038 }
1039
1040 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1041
1042 sub print_drive {
1043     my ($vmid, $drive) = @_;
1044
1045     my $opts = '';
1046     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
1047         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1048     }
1049
1050     if ($drive->{size}) {
1051         $opts .= ",size=" . &$format_size($drive->{size});
1052     }
1053
1054     return "$drive->{file}$opts";
1055 }
1056
1057 sub scsi_inquiry {
1058     my($fh, $noerr) = @_;
1059
1060     my $SG_IO = 0x2285;
1061     my $SG_GET_VERSION_NUM = 0x2282;
1062
1063     my $versionbuf = "\x00" x 8;
1064     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1065     if (!$ret) {
1066         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1067         return undef;
1068     }
1069     my $version = unpack("I", $versionbuf);
1070     if ($version < 30000) {
1071         die "scsi generic interface too old\n"  if !$noerr;
1072         return undef;
1073     }
1074
1075     my $buf = "\x00" x 36;
1076     my $sensebuf = "\x00" x 8;
1077     my $cmd = pack("C x3 C x1", 0x12, 36);
1078
1079     # see /usr/include/scsi/sg.h
1080     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1081
1082     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1083                       length($sensebuf), 0, length($buf), $buf,
1084                       $cmd, $sensebuf, 6000);
1085
1086     $ret = ioctl($fh, $SG_IO, $packet);
1087     if (!$ret) {
1088         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1089         return undef;
1090     }
1091
1092     my @res = unpack($sg_io_hdr_t, $packet);
1093     if ($res[17] || $res[18]) {
1094         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1095         return undef;
1096     }
1097
1098     my $res = {};
1099     (my $byte0, my $byte1, $res->{vendor},
1100      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1101
1102     $res->{removable} = $byte1 & 128 ? 1 : 0;
1103     $res->{type} = $byte0 & 31;
1104
1105     return $res;
1106 }
1107
1108 sub path_is_scsi {
1109     my ($path) = @_;
1110
1111     my $fh = IO::File->new("+<$path") || return undef;
1112     my $res = scsi_inquiry($fh, 1);
1113     close($fh);
1114
1115     return $res;
1116 }
1117
1118 sub machine_type_is_q35 {
1119     my ($conf) = @_;
1120
1121     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1122 }
1123
1124 sub print_tabletdevice_full {
1125     my ($conf) = @_;
1126
1127     my $q35 = machine_type_is_q35($conf);
1128
1129     # we use uhci for old VMs because tablet driver was buggy in older qemu
1130     my $usbbus = $q35 ? "ehci" : "uhci";
1131
1132     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1133 }
1134
1135 sub print_drivedevice_full {
1136     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1137
1138     my $device = '';
1139     my $maxdev = 0;
1140
1141     if ($drive->{interface} eq 'virtio') {
1142         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1143         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1144         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1145     } elsif ($drive->{interface} eq 'scsi') {
1146
1147         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1148         my $unit = $drive->{index} % $maxdev;
1149         my $devicetype = 'hd';
1150         my $path = '';
1151         if (drive_is_cdrom($drive)) {
1152               $devicetype = 'cd';
1153         } else {
1154               if ($drive->{file} =~ m|^/|) {
1155                   $path = $drive->{file};
1156               } else {
1157                   $path = PVE::Storage::path($storecfg, $drive->{file});
1158               }
1159
1160               if($path =~ m/^iscsi\:\/\//){
1161                   $devicetype = 'generic';
1162               } else {
1163                   if (my $info = path_is_scsi($path)) {
1164                       if ($info->{type} == 0) {
1165                           $devicetype = 'block';
1166                       } elsif ($info->{type} == 1) { # tape
1167                           $devicetype = 'generic';
1168                       }
1169                   }
1170               }
1171          }
1172
1173         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1174             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1175         } else {
1176             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1177         }
1178
1179     } elsif ($drive->{interface} eq 'ide'){
1180         $maxdev = 2;
1181         my $controller = int($drive->{index} / $maxdev);
1182         my $unit = $drive->{index} % $maxdev;
1183         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1184
1185         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1186     } elsif ($drive->{interface} eq 'sata'){
1187         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1188         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1189         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1190     } elsif ($drive->{interface} eq 'usb') {
1191         die "implement me";
1192         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1193     } else {
1194         die "unsupported interface type";
1195     }
1196
1197     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1198
1199     return $device;
1200 }
1201
1202 sub get_initiator_name {
1203     my $initiator;
1204
1205     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1206     while (defined(my $line = <$fh>)) {
1207         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1208         $initiator = $1;
1209         last;
1210     }
1211     $fh->close();
1212
1213     return $initiator;
1214 }
1215
1216 sub print_drive_full {
1217     my ($storecfg, $vmid, $drive) = @_;
1218
1219     my $opts = '';
1220     foreach my $o (@qemu_drive_options) {
1221         next if $o eq 'bootindex';
1222         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1223     }
1224
1225     foreach my $o (qw(bps bps_rd bps_wr)) {
1226         my $v = $drive->{"m$o"};
1227         $opts .= ",$o=" . int($v*1024*1024) if $v;
1228     }
1229
1230     # aio native works only with O_DIRECT
1231     if (!$drive->{aio}) {
1232         if(!$drive->{cache} || $drive->{cache} eq 'none' || $drive->{cache} eq 'directsync') {
1233             $opts .= ",aio=native";
1234         } else {
1235             $opts .= ",aio=threads";
1236         }
1237     }
1238
1239
1240     my $path;
1241     my $volid = $drive->{file};
1242     if (drive_is_cdrom($drive)) {
1243         $path = get_iso_path($storecfg, $vmid, $volid);
1244     } else {
1245         if ($volid =~ m|^/|) {
1246             $path = $volid;
1247         } else {
1248             $path = PVE::Storage::path($storecfg, $volid);
1249         }
1250     }
1251
1252     $opts .= ",cache=none" if !$drive->{cache} && !drive_is_cdrom($drive);
1253
1254     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1255     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1256
1257     my $pathinfo = $path ? "file=$path," : '';
1258
1259     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1260 }
1261
1262 sub print_netdevice_full {
1263     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1264
1265     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1266
1267     my $device = $net->{model};
1268     if ($net->{model} eq 'virtio') {
1269          $device = 'virtio-net-pci';
1270      };
1271
1272     my $pciaddr = print_pci_addr("$netid", $bridges);
1273     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1274     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1275         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1276         my $vectors = $net->{queues} * 2 + 2;
1277         $tmpstr .= ",vectors=$vectors,mq=on";
1278     }
1279     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1280     return $tmpstr;
1281 }
1282
1283 sub print_netdev_full {
1284     my ($vmid, $conf, $net, $netid) = @_;
1285
1286     my $i = '';
1287     if ($netid =~ m/^net(\d+)$/) {
1288         $i = int($1);
1289     }
1290
1291     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1292
1293     my $ifname = "tap${vmid}i$i";
1294
1295     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1296     die "interface name '$ifname' is too long (max 15 character)\n"
1297         if length($ifname) >= 16;
1298
1299     my $vhostparam = '';
1300     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1301
1302     my $vmname = $conf->{name} || "vm$vmid";
1303
1304     my $netdev = "";
1305
1306     if ($net->{bridge}) {
1307         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1308     } else {
1309         $netdev = "type=user,id=$netid,hostname=$vmname";
1310     }
1311
1312     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1313
1314     return $netdev;
1315 }
1316
1317 sub drive_is_cdrom {
1318     my ($drive) = @_;
1319
1320     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1321
1322 }
1323
1324 sub parse_numa {
1325     my ($data) = @_;
1326
1327     my $res = {};
1328
1329     foreach my $kvp (split(/,/, $data)) {
1330
1331         if ($kvp =~ m/^memory=(\S+)$/) {
1332             $res->{memory} = $1;
1333         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1334             $res->{policy} = $1;
1335         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1336             $res->{cpus}->{start} = $1;
1337             $res->{cpus}->{end} = $3;
1338         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1339             $res->{hostnodes}->{start} = $1;
1340             $res->{hostnodes}->{end} = $3;
1341         } else {
1342             return undef;
1343         }
1344     }
1345
1346     return $res;
1347 }
1348
1349 sub parse_hostpci {
1350     my ($value) = @_;
1351
1352     return undef if !$value;
1353
1354
1355     my @list = split(/,/, $value);
1356     my $found;
1357
1358     my $res = {};
1359     foreach my $kv (@list) {
1360
1361         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1362             $found = 1;
1363             if(defined($4)){
1364                 push @{$res->{pciid}}, { id => $2 , function => $4};
1365
1366             }else{
1367                 my $pcidevices = lspci($2);
1368                 $res->{pciid} = $pcidevices->{$2};
1369             }
1370         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1371             $res->{driver} = $1;
1372         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1373             $res->{rombar} = $1;
1374         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1375             $res->{'x-vga'} = $1;
1376         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1377             $res->{pcie} = 1 if $1 == 1;
1378         } else {
1379             warn "unknown hostpci setting '$kv'\n";
1380         }
1381     }
1382
1383     return undef if !$found;
1384
1385     return $res;
1386 }
1387
1388 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1389 sub parse_net {
1390     my ($data) = @_;
1391
1392     my $res = {};
1393
1394     foreach my $kvp (split(/,/, $data)) {
1395
1396         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1397             my $model = lc($1);
1398             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1399             $res->{model} = $model;
1400             $res->{macaddr} = $mac;
1401         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1402             $res->{bridge} = $1;
1403         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1404             $res->{queues} = $1;
1405         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1406             $res->{rate} = $1;
1407         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1408             $res->{tag} = $1;
1409         } elsif ($kvp =~ m/^firewall=([01])$/) {
1410             $res->{firewall} = $1;
1411         } elsif ($kvp =~ m/^link_down=([01])$/) {
1412             $res->{link_down} = $1;
1413         } else {
1414             return undef;
1415         }
1416
1417     }
1418
1419     return undef if !$res->{model};
1420
1421     return $res;
1422 }
1423
1424 sub print_net {
1425     my $net = shift;
1426
1427     my $res = "$net->{model}";
1428     $res .= "=$net->{macaddr}" if $net->{macaddr};
1429     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1430     $res .= ",rate=$net->{rate}" if $net->{rate};
1431     $res .= ",tag=$net->{tag}" if $net->{tag};
1432     $res .= ",firewall=1" if $net->{firewall};
1433     $res .= ",link_down=1" if $net->{link_down};
1434     $res .= ",queues=$net->{queues}" if $net->{queues};
1435
1436     return $res;
1437 }
1438
1439 sub add_random_macs {
1440     my ($settings) = @_;
1441
1442     foreach my $opt (keys %$settings) {
1443         next if $opt !~ m/^net(\d+)$/;
1444         my $net = parse_net($settings->{$opt});
1445         next if !$net;
1446         $settings->{$opt} = print_net($net);
1447     }
1448 }
1449
1450 sub add_unused_volume {
1451     my ($config, $volid) = @_;
1452
1453     my $key;
1454     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1455         my $test = "unused$ind";
1456         if (my $vid = $config->{$test}) {
1457             return if $vid eq $volid; # do not add duplicates
1458         } else {
1459             $key = $test;
1460         }
1461     }
1462
1463     die "To many unused volume - please delete them first.\n" if !$key;
1464
1465     $config->{$key} = $volid;
1466
1467     return $key;
1468 }
1469
1470 sub vm_is_volid_owner {
1471     my ($storecfg, $vmid, $volid) = @_;
1472
1473     if ($volid !~  m|^/|) {
1474         my ($path, $owner);
1475         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1476         if ($owner && ($owner == $vmid)) {
1477             return 1;
1478         }
1479     }
1480
1481     return undef;
1482 }
1483
1484 sub vmconfig_delete_pending_option {
1485     my ($conf, $key) = @_;
1486
1487     delete $conf->{pending}->{$key};
1488     my $pending_delete_hash = { $key => 1 };
1489     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1490         $pending_delete_hash->{$opt} = 1;
1491     }
1492     $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
1493 }
1494
1495 sub vmconfig_undelete_pending_option {
1496     my ($conf, $key) = @_;
1497
1498     my $pending_delete_hash = {};
1499     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1500         $pending_delete_hash->{$opt} = 1;
1501     }
1502     delete $pending_delete_hash->{$key};
1503
1504     my @keylist = keys %$pending_delete_hash;
1505     if (scalar(@keylist)) {
1506         $conf->{pending}->{delete} = join(',', @keylist);
1507     } else {
1508         delete $conf->{pending}->{delete};
1509     }
1510 }
1511
1512 sub vmconfig_register_unused_drive {
1513     my ($storecfg, $vmid, $conf, $drive) = @_;
1514
1515     if (!drive_is_cdrom($drive)) {
1516         my $volid = $drive->{file};
1517         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1518             add_unused_volume($conf, $volid, $vmid);
1519         }
1520     }
1521 }
1522
1523 sub vmconfig_cleanup_pending {
1524     my ($conf) = @_;
1525
1526     # remove pending changes when nothing changed
1527     my $changes;
1528     foreach my $opt (keys %{$conf->{pending}}) {
1529         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1530             $changes = 1;
1531             delete $conf->{pending}->{$opt};
1532         }
1533     }
1534
1535     # remove delete if option is not set
1536     my $pending_delete_hash = {};
1537     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1538         if (defined($conf->{$opt})) {
1539             $pending_delete_hash->{$opt} = 1;
1540         } else {
1541             $changes = 1;
1542         }
1543     }
1544
1545     my @keylist = keys %$pending_delete_hash;
1546     if (scalar(@keylist)) {
1547         $conf->{pending}->{delete} = join(',', @keylist);
1548     } else {
1549         delete $conf->{pending}->{delete};
1550     }
1551
1552     return $changes;
1553 }
1554
1555 my $valid_smbios1_options = {
1556     manufacturer => '\S+',
1557     product => '\S+',
1558     version => '\S+',
1559     serial => '\S+',
1560     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1561     sku => '\S+',
1562     family => '\S+',
1563 };
1564
1565 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1566 sub parse_smbios1 {
1567     my ($data) = @_;
1568
1569     my $res = {};
1570
1571     foreach my $kvp (split(/,/, $data)) {
1572         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1573         my ($k, $v) = split(/=/, $kvp);
1574         return undef if !defined($k) || !defined($v);
1575         return undef if !$valid_smbios1_options->{$k};
1576         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1577         $res->{$k} = $v;
1578     }
1579
1580     return $res;
1581 }
1582
1583 sub print_smbios1 {
1584     my ($smbios1) = @_;
1585
1586     my $data = '';
1587     foreach my $k (keys %$smbios1) {
1588         next if !defined($smbios1->{$k});
1589         next if !$valid_smbios1_options->{$k};
1590         $data .= ',' if $data;
1591         $data .= "$k=$smbios1->{$k}";
1592     }
1593     return $data;
1594 }
1595
1596 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1597 sub verify_smbios1 {
1598     my ($value, $noerr) = @_;
1599
1600     return $value if parse_smbios1($value);
1601
1602     return undef if $noerr;
1603
1604     die "unable to parse smbios (type 1) options\n";
1605 }
1606
1607 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1608 sub verify_bootdisk {
1609     my ($value, $noerr) = @_;
1610
1611     return $value if valid_drivename($value);
1612
1613     return undef if $noerr;
1614
1615     die "invalid boot disk '$value'\n";
1616 }
1617
1618 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1619 sub verify_numa {
1620     my ($value, $noerr) = @_;
1621
1622     return $value if parse_numa($value);
1623
1624     return undef if $noerr;
1625
1626     die "unable to parse numa options\n";
1627 }
1628
1629 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1630 sub verify_net {
1631     my ($value, $noerr) = @_;
1632
1633     return $value if parse_net($value);
1634
1635     return undef if $noerr;
1636
1637     die "unable to parse network options\n";
1638 }
1639
1640 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1641 sub verify_drive {
1642     my ($value, $noerr) = @_;
1643
1644     return $value if parse_drive(undef, $value);
1645
1646     return undef if $noerr;
1647
1648     die "unable to parse drive options\n";
1649 }
1650
1651 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1652 sub verify_hostpci {
1653     my ($value, $noerr) = @_;
1654
1655     return $value if parse_hostpci($value);
1656
1657     return undef if $noerr;
1658
1659     die "unable to parse pci id\n";
1660 }
1661
1662 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1663 sub verify_watchdog {
1664     my ($value, $noerr) = @_;
1665
1666     return $value if parse_watchdog($value);
1667
1668     return undef if $noerr;
1669
1670     die "unable to parse watchdog options\n";
1671 }
1672
1673 sub parse_watchdog {
1674     my ($value) = @_;
1675
1676     return undef if !$value;
1677
1678     my $res = {};
1679
1680     foreach my $p (split(/,/, $value)) {
1681         next if $p =~ m/^\s*$/;
1682
1683         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1684             $res->{model} = $2;
1685         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1686             $res->{action} = $2;
1687         } else {
1688             return undef;
1689         }
1690     }
1691
1692     return $res;
1693 }
1694
1695 sub parse_usb_device {
1696     my ($value) = @_;
1697
1698     return undef if !$value;
1699
1700     my @dl = split(/,/, $value);
1701     my $found;
1702
1703     my $res = {};
1704     foreach my $v (@dl) {
1705         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1706             $found = 1;
1707             $res->{vendorid} = $2;
1708             $res->{productid} = $4;
1709         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1710             $found = 1;
1711             $res->{hostbus} = $1;
1712             $res->{hostport} = $2;
1713         } elsif ($v =~ m/^spice$/) {
1714             $found = 1;
1715             $res->{spice} = 1;
1716         } else {
1717             return undef;
1718         }
1719     }
1720     return undef if !$found;
1721
1722     return $res;
1723 }
1724
1725 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1726 sub verify_usb_device {
1727     my ($value, $noerr) = @_;
1728
1729     return $value if parse_usb_device($value);
1730
1731     return undef if $noerr;
1732
1733     die "unable to parse usb device\n";
1734 }
1735
1736 # add JSON properties for create and set function
1737 sub json_config_properties {
1738     my $prop = shift;
1739
1740     foreach my $opt (keys %$confdesc) {
1741         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1742         $prop->{$opt} = $confdesc->{$opt};
1743     }
1744
1745     return $prop;
1746 }
1747
1748 sub check_type {
1749     my ($key, $value) = @_;
1750
1751     die "unknown setting '$key'\n" if !$confdesc->{$key};
1752
1753     my $type = $confdesc->{$key}->{type};
1754
1755     if (!defined($value)) {
1756         die "got undefined value\n";
1757     }
1758
1759     if ($value =~ m/[\n\r]/) {
1760         die "property contains a line feed\n";
1761     }
1762
1763     if ($type eq 'boolean') {
1764         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1765         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1766         die "type check ('boolean') failed - got '$value'\n";
1767     } elsif ($type eq 'integer') {
1768         return int($1) if $value =~ m/^(\d+)$/;
1769         die "type check ('integer') failed - got '$value'\n";
1770     } elsif ($type eq 'number') {
1771         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1772         die "type check ('number') failed - got '$value'\n";
1773     } elsif ($type eq 'string') {
1774         if (my $fmt = $confdesc->{$key}->{format}) {
1775             if ($fmt eq 'pve-qm-drive') {
1776                 # special case - we need to pass $key to parse_drive()
1777                 my $drive = parse_drive($key, $value);
1778                 return $value if $drive;
1779                 die "unable to parse drive options\n";
1780             }
1781             PVE::JSONSchema::check_format($fmt, $value);
1782             return $value;
1783         }
1784         $value =~ s/^\"(.*)\"$/$1/;
1785         return $value;
1786     } else {
1787         die "internal error"
1788     }
1789 }
1790
1791 sub lock_config_full {
1792     my ($vmid, $timeout, $code, @param) = @_;
1793
1794     my $filename = config_file_lock($vmid);
1795
1796     my $res = lock_file($filename, $timeout, $code, @param);
1797
1798     die $@ if $@;
1799
1800     return $res;
1801 }
1802
1803 sub lock_config_mode {
1804     my ($vmid, $timeout, $shared, $code, @param) = @_;
1805
1806     my $filename = config_file_lock($vmid);
1807
1808     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1809
1810     die $@ if $@;
1811
1812     return $res;
1813 }
1814
1815 sub lock_config {
1816     my ($vmid, $code, @param) = @_;
1817
1818     return lock_config_full($vmid, 10, $code, @param);
1819 }
1820
1821 sub cfs_config_path {
1822     my ($vmid, $node) = @_;
1823
1824     $node = $nodename if !$node;
1825     return "nodes/$node/qemu-server/$vmid.conf";
1826 }
1827
1828 sub check_iommu_support{
1829     #fixme : need to check IOMMU support
1830     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1831
1832     my $iommu=1;
1833     return $iommu;
1834
1835 }
1836
1837 sub config_file {
1838     my ($vmid, $node) = @_;
1839
1840     my $cfspath = cfs_config_path($vmid, $node);
1841     return "/etc/pve/$cfspath";
1842 }
1843
1844 sub config_file_lock {
1845     my ($vmid) = @_;
1846
1847     return "$lock_dir/lock-$vmid.conf";
1848 }
1849
1850 sub touch_config {
1851     my ($vmid) = @_;
1852
1853     my $conf = config_file($vmid);
1854     utime undef, undef, $conf;
1855 }
1856
1857 sub destroy_vm {
1858     my ($storecfg, $vmid, $keep_empty_config) = @_;
1859
1860     my $conffile = config_file($vmid);
1861
1862     my $conf = load_config($vmid);
1863
1864     check_lock($conf);
1865
1866     # only remove disks owned by this VM
1867     foreach_drive($conf, sub {
1868         my ($ds, $drive) = @_;
1869
1870         return if drive_is_cdrom($drive);
1871
1872         my $volid = $drive->{file};
1873
1874         return if !$volid || $volid =~ m|^/|;
1875
1876         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1877         return if !$path || !$owner || ($owner != $vmid);
1878
1879         PVE::Storage::vdisk_free($storecfg, $volid);
1880     });
1881
1882     if ($keep_empty_config) {
1883         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1884     } else {
1885         unlink $conffile;
1886     }
1887
1888     # also remove unused disk
1889     eval {
1890         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1891
1892         eval {
1893             PVE::Storage::foreach_volid($dl, sub {
1894                 my ($volid, $sid, $volname, $d) = @_;
1895                 PVE::Storage::vdisk_free($storecfg, $volid);
1896             });
1897         };
1898         warn $@ if $@;
1899
1900     };
1901     warn $@ if $@;
1902 }
1903
1904 sub load_config {
1905     my ($vmid, $node) = @_;
1906
1907     my $cfspath = cfs_config_path($vmid, $node);
1908
1909     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1910
1911     die "no such VM ('$vmid')\n" if !defined($conf);
1912
1913     return $conf;
1914 }
1915
1916 sub parse_vm_config {
1917     my ($filename, $raw) = @_;
1918
1919     return undef if !defined($raw);
1920
1921     my $res = {
1922         digest => Digest::SHA::sha1_hex($raw),
1923         snapshots => {},
1924         pending => {},
1925     };
1926
1927     $filename =~ m|/qemu-server/(\d+)\.conf$|
1928         || die "got strange filename '$filename'";
1929
1930     my $vmid = $1;
1931
1932     my $conf = $res;
1933     my $descr = '';
1934     my $section = '';
1935
1936     my @lines = split(/\n/, $raw);
1937     foreach my $line (@lines) {
1938         next if $line =~ m/^\s*$/;
1939
1940         if ($line =~ m/^\[PENDING\]\s*$/i) {
1941             $section = 'pending';
1942             $conf->{description} = $descr if $descr;
1943             $descr = '';
1944             $conf = $res->{$section} = {};
1945             next;
1946
1947         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1948             $section = $1;
1949             $conf->{description} = $descr if $descr;
1950             $descr = '';
1951             $conf = $res->{snapshots}->{$section} = {};
1952             next;
1953         }
1954
1955         if ($line =~ m/^\#(.*)\s*$/) {
1956             $descr .= PVE::Tools::decode_text($1) . "\n";
1957             next;
1958         }
1959
1960         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1961             $descr .= PVE::Tools::decode_text($2);
1962         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1963             $conf->{snapstate} = $1;
1964         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1965             my $key = $1;
1966             my $value = $2;
1967             $conf->{$key} = $value;
1968         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1969             my $value = $1;
1970             if ($section eq 'pending') {
1971                 $conf->{delete} = $value; # we parse this later
1972             } else {
1973                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1974             }
1975         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1976             my $key = $1;
1977             my $value = $2;
1978             eval { $value = check_type($key, $value); };
1979             if ($@) {
1980                 warn "vm $vmid - unable to parse value of '$key' - $@";
1981             } else {
1982                 my $fmt = $confdesc->{$key}->{format};
1983                 if ($fmt && $fmt eq 'pve-qm-drive') {
1984                     my $v = parse_drive($key, $value);
1985                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1986                         $v->{file} = $volid;
1987                         $value = print_drive($vmid, $v);
1988                     } else {
1989                         warn "vm $vmid - unable to parse value of '$key'\n";
1990                         next;
1991                     }
1992                 }
1993
1994                 if ($key eq 'cdrom') {
1995                     $conf->{ide2} = $value;
1996                 } else {
1997                     $conf->{$key} = $value;
1998                 }
1999             }
2000         }
2001     }
2002
2003     $conf->{description} = $descr if $descr;
2004
2005     delete $res->{snapstate}; # just to be sure
2006
2007     return $res;
2008 }
2009
2010 sub write_vm_config {
2011     my ($filename, $conf) = @_;
2012
2013     delete $conf->{snapstate}; # just to be sure
2014
2015     if ($conf->{cdrom}) {
2016         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2017         $conf->{ide2} = $conf->{cdrom};
2018         delete $conf->{cdrom};
2019     }
2020
2021     # we do not use 'smp' any longer
2022     if ($conf->{sockets}) {
2023         delete $conf->{smp};
2024     } elsif ($conf->{smp}) {
2025         $conf->{sockets} = $conf->{smp};
2026         delete $conf->{cores};
2027         delete $conf->{smp};
2028     }
2029
2030     my $used_volids = {};
2031
2032     my $cleanup_config = sub {
2033         my ($cref, $pending, $snapname) = @_;
2034
2035         foreach my $key (keys %$cref) {
2036             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2037                 $key eq 'snapstate' || $key eq 'pending';
2038             my $value = $cref->{$key};
2039             if ($key eq 'delete') {
2040                 die "propertry 'delete' is only allowed in [PENDING]\n"
2041                     if !$pending;
2042                 # fixme: check syntax?
2043                 next;
2044             }
2045             eval { $value = check_type($key, $value); };
2046             die "unable to parse value of '$key' - $@" if $@;
2047
2048             $cref->{$key} = $value;
2049
2050             if (!$snapname && valid_drivename($key)) {
2051                 my $drive = parse_drive($key, $value);
2052                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2053             }
2054         }
2055     };
2056
2057     &$cleanup_config($conf);
2058
2059     &$cleanup_config($conf->{pending}, 1);
2060
2061     foreach my $snapname (keys %{$conf->{snapshots}}) {
2062         die "internal error" if $snapname eq 'pending';
2063         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2064     }
2065
2066     # remove 'unusedX' settings if we re-add a volume
2067     foreach my $key (keys %$conf) {
2068         my $value = $conf->{$key};
2069         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2070             delete $conf->{$key};
2071         }
2072     }
2073
2074     my $generate_raw_config = sub {
2075         my ($conf) = @_;
2076
2077         my $raw = '';
2078
2079         # add description as comment to top of file
2080         my $descr = $conf->{description} || '';
2081         foreach my $cl (split(/\n/, $descr)) {
2082             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2083         }
2084
2085         foreach my $key (sort keys %$conf) {
2086             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2087             $raw .= "$key: $conf->{$key}\n";
2088         }
2089         return $raw;
2090     };
2091
2092     my $raw = &$generate_raw_config($conf);
2093
2094     if (scalar(keys %{$conf->{pending}})){
2095         $raw .= "\n[PENDING]\n";
2096         $raw .= &$generate_raw_config($conf->{pending});
2097     }
2098
2099     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2100         $raw .= "\n[$snapname]\n";
2101         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2102     }
2103
2104     return $raw;
2105 }
2106
2107 sub update_config_nolock {
2108     my ($vmid, $conf, $skiplock) = @_;
2109
2110     check_lock($conf) if !$skiplock;
2111
2112     my $cfspath = cfs_config_path($vmid);
2113
2114     PVE::Cluster::cfs_write_file($cfspath, $conf);
2115 }
2116
2117 sub update_config {
2118     my ($vmid, $conf, $skiplock) = @_;
2119
2120     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2121 }
2122
2123 sub load_defaults {
2124
2125     my $res = {};
2126
2127     # we use static defaults from our JSON schema configuration
2128     foreach my $key (keys %$confdesc) {
2129         if (defined(my $default = $confdesc->{$key}->{default})) {
2130             $res->{$key} = $default;
2131         }
2132     }
2133
2134     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2135     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2136
2137     return $res;
2138 }
2139
2140 sub config_list {
2141     my $vmlist = PVE::Cluster::get_vmlist();
2142     my $res = {};
2143     return $res if !$vmlist || !$vmlist->{ids};
2144     my $ids = $vmlist->{ids};
2145
2146     foreach my $vmid (keys %$ids) {
2147         my $d = $ids->{$vmid};
2148         next if !$d->{node} || $d->{node} ne $nodename;
2149         next if !$d->{type} || $d->{type} ne 'qemu';
2150         $res->{$vmid}->{exists} = 1;
2151     }
2152     return $res;
2153 }
2154
2155 # test if VM uses local resources (to prevent migration)
2156 sub check_local_resources {
2157     my ($conf, $noerr) = @_;
2158
2159     my $loc_res = 0;
2160
2161     $loc_res = 1 if $conf->{hostusb}; # old syntax
2162     $loc_res = 1 if $conf->{hostpci}; # old syntax
2163
2164     foreach my $k (keys %$conf) {
2165         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2166         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2167     }
2168
2169     die "VM uses local resources\n" if $loc_res && !$noerr;
2170
2171     return $loc_res;
2172 }
2173
2174 # check if used storages are available on all nodes (use by migrate)
2175 sub check_storage_availability {
2176     my ($storecfg, $conf, $node) = @_;
2177
2178     foreach_drive($conf, sub {
2179         my ($ds, $drive) = @_;
2180
2181         my $volid = $drive->{file};
2182         return if !$volid;
2183
2184         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2185         return if !$sid;
2186
2187         # check if storage is available on both nodes
2188         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2189         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2190    });
2191 }
2192
2193 # list nodes where all VM images are available (used by has_feature API)
2194 sub shared_nodes {
2195     my ($conf, $storecfg) = @_;
2196
2197     my $nodelist = PVE::Cluster::get_nodelist();
2198     my $nodehash = { map { $_ => 1 } @$nodelist };
2199     my $nodename = PVE::INotify::nodename();
2200
2201     foreach_drive($conf, sub {
2202         my ($ds, $drive) = @_;
2203
2204         my $volid = $drive->{file};
2205         return if !$volid;
2206
2207         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2208         if ($storeid) {
2209             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2210             if ($scfg->{disable}) {
2211                 $nodehash = {};
2212             } elsif (my $avail = $scfg->{nodes}) {
2213                 foreach my $node (keys %$nodehash) {
2214                     delete $nodehash->{$node} if !$avail->{$node};
2215                 }
2216             } elsif (!$scfg->{shared}) {
2217                 foreach my $node (keys %$nodehash) {
2218                     delete $nodehash->{$node} if $node ne $nodename
2219                 }
2220             }
2221         }
2222     });
2223
2224     return $nodehash
2225 }
2226
2227 sub check_lock {
2228     my ($conf) = @_;
2229
2230     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2231 }
2232
2233 sub check_cmdline {
2234     my ($pidfile, $pid) = @_;
2235
2236     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2237     if (defined($fh)) {
2238         my $line = <$fh>;
2239         $fh->close;
2240         return undef if !$line;
2241         my @param = split(/\0/, $line);
2242
2243         my $cmd = $param[0];
2244         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2245
2246         for (my $i = 0; $i < scalar (@param); $i++) {
2247             my $p = $param[$i];
2248             next if !$p;
2249             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2250                 my $p = $param[$i+1];
2251                 return 1 if $p && ($p eq $pidfile);
2252                 return undef;
2253             }
2254         }
2255     }
2256     return undef;
2257 }
2258
2259 sub check_running {
2260     my ($vmid, $nocheck, $node) = @_;
2261
2262     my $filename = config_file($vmid, $node);
2263
2264     die "unable to find configuration file for VM $vmid - no such machine\n"
2265         if !$nocheck && ! -f $filename;
2266
2267     my $pidfile = pidfile_name($vmid);
2268
2269     if (my $fd = IO::File->new("<$pidfile")) {
2270         my $st = stat($fd);
2271         my $line = <$fd>;
2272         close($fd);
2273
2274         my $mtime = $st->mtime;
2275         if ($mtime > time()) {
2276             warn "file '$filename' modified in future\n";
2277         }
2278
2279         if ($line =~ m/^(\d+)$/) {
2280             my $pid = $1;
2281             if (check_cmdline($pidfile, $pid)) {
2282                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2283                     return $pid;
2284                 }
2285             }
2286         }
2287     }
2288
2289     return undef;
2290 }
2291
2292 sub vzlist {
2293
2294     my $vzlist = config_list();
2295
2296     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2297
2298     while (defined(my $de = $fd->read)) {
2299         next if $de !~ m/^(\d+)\.pid$/;
2300         my $vmid = $1;
2301         next if !defined($vzlist->{$vmid});
2302         if (my $pid = check_running($vmid)) {
2303             $vzlist->{$vmid}->{pid} = $pid;
2304         }
2305     }
2306
2307     return $vzlist;
2308 }
2309
2310 sub disksize {
2311     my ($storecfg, $conf) = @_;
2312
2313     my $bootdisk = $conf->{bootdisk};
2314     return undef if !$bootdisk;
2315     return undef if !valid_drivename($bootdisk);
2316
2317     return undef if !$conf->{$bootdisk};
2318
2319     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2320     return undef if !defined($drive);
2321
2322     return undef if drive_is_cdrom($drive);
2323
2324     my $volid = $drive->{file};
2325     return undef if !$volid;
2326
2327     return $drive->{size};
2328 }
2329
2330 my $last_proc_pid_stat;
2331
2332 # get VM status information
2333 # This must be fast and should not block ($full == false)
2334 # We only query KVM using QMP if $full == true (this can be slow)
2335 sub vmstatus {
2336     my ($opt_vmid, $full) = @_;
2337
2338     my $res = {};
2339
2340     my $storecfg = PVE::Storage::config();
2341
2342     my $list = vzlist();
2343     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2344
2345     my $cpucount = $cpuinfo->{cpus} || 1;
2346
2347     foreach my $vmid (keys %$list) {
2348         next if $opt_vmid && ($vmid ne $opt_vmid);
2349
2350         my $cfspath = cfs_config_path($vmid);
2351         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2352
2353         my $d = {};
2354         $d->{pid} = $list->{$vmid}->{pid};
2355
2356         # fixme: better status?
2357         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2358
2359         my $size = disksize($storecfg, $conf);
2360         if (defined($size)) {
2361             $d->{disk} = 0; # no info available
2362             $d->{maxdisk} = $size;
2363         } else {
2364             $d->{disk} = 0;
2365             $d->{maxdisk} = 0;
2366         }
2367
2368         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2369         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2370         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2371
2372         $d->{name} = $conf->{name} || "VM $vmid";
2373         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2374
2375         if ($conf->{balloon}) {
2376             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2377             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2378         }
2379
2380         $d->{uptime} = 0;
2381         $d->{cpu} = 0;
2382         $d->{mem} = 0;
2383
2384         $d->{netout} = 0;
2385         $d->{netin} = 0;
2386
2387         $d->{diskread} = 0;
2388         $d->{diskwrite} = 0;
2389
2390         $d->{template} = is_template($conf);
2391
2392         $res->{$vmid} = $d;
2393     }
2394
2395     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2396     foreach my $dev (keys %$netdev) {
2397         next if $dev !~ m/^tap([1-9]\d*)i/;
2398         my $vmid = $1;
2399         my $d = $res->{$vmid};
2400         next if !$d;
2401
2402         $d->{netout} += $netdev->{$dev}->{receive};
2403         $d->{netin} += $netdev->{$dev}->{transmit};
2404     }
2405
2406     my $ctime = gettimeofday;
2407
2408     foreach my $vmid (keys %$list) {
2409
2410         my $d = $res->{$vmid};
2411         my $pid = $d->{pid};
2412         next if !$pid;
2413
2414         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2415         next if !$pstat; # not running
2416
2417         my $used = $pstat->{utime} + $pstat->{stime};
2418
2419         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2420
2421         if ($pstat->{vsize}) {
2422             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2423         }
2424
2425         my $old = $last_proc_pid_stat->{$pid};
2426         if (!$old) {
2427             $last_proc_pid_stat->{$pid} = {
2428                 time => $ctime,
2429                 used => $used,
2430                 cpu => 0,
2431             };
2432             next;
2433         }
2434
2435         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2436
2437         if ($dtime > 1000) {
2438             my $dutime = $used -  $old->{used};
2439
2440             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2441             $last_proc_pid_stat->{$pid} = {
2442                 time => $ctime,
2443                 used => $used,
2444                 cpu => $d->{cpu},
2445             };
2446         } else {
2447             $d->{cpu} = $old->{cpu};
2448         }
2449     }
2450
2451     return $res if !$full;
2452
2453     my $qmpclient = PVE::QMPClient->new();
2454
2455     my $ballooncb = sub {
2456         my ($vmid, $resp) = @_;
2457
2458         my $info = $resp->{'return'};
2459         return if !$info->{max_mem};
2460
2461         my $d = $res->{$vmid};
2462
2463         # use memory assigned to VM
2464         $d->{maxmem} = $info->{max_mem};
2465         $d->{balloon} = $info->{actual};
2466
2467         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2468             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2469             $d->{freemem} = $info->{free_mem};
2470         }
2471
2472     };
2473
2474     my $blockstatscb = sub {
2475         my ($vmid, $resp) = @_;
2476         my $data = $resp->{'return'} || [];
2477         my $totalrdbytes = 0;
2478         my $totalwrbytes = 0;
2479         for my $blockstat (@$data) {
2480             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2481             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2482         }
2483         $res->{$vmid}->{diskread} = $totalrdbytes;
2484         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2485     };
2486
2487     my $statuscb = sub {
2488         my ($vmid, $resp) = @_;
2489
2490         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2491         # this fails if ballon driver is not loaded, so this must be
2492         # the last commnand (following command are aborted if this fails).
2493         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2494
2495         my $status = 'unknown';
2496         if (!defined($status = $resp->{'return'}->{status})) {
2497             warn "unable to get VM status\n";
2498             return;
2499         }
2500
2501         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2502     };
2503
2504     foreach my $vmid (keys %$list) {
2505         next if $opt_vmid && ($vmid ne $opt_vmid);
2506         next if !$res->{$vmid}->{pid}; # not running
2507         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2508     }
2509
2510     $qmpclient->queue_execute(undef, 1);
2511
2512     foreach my $vmid (keys %$list) {
2513         next if $opt_vmid && ($vmid ne $opt_vmid);
2514         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2515     }
2516
2517     return $res;
2518 }
2519
2520 sub foreach_dimm {
2521     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2522
2523     my $dimm_id = 0;
2524     my $current_size = 1024;
2525     my $dimm_size = 512;
2526     return if $current_size == $memory;
2527
2528     for (my $j = 0; $j < 8; $j++) {
2529         for (my $i = 0; $i < 32; $i++) {
2530             my $name = "dimm${dimm_id}";
2531             $dimm_id++;
2532             my $numanode = $i % $sockets;
2533             $current_size += $dimm_size;
2534             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2535             return  $current_size if $current_size >= $memory;
2536         }
2537         $dimm_size *= 2;
2538     }
2539 }
2540
2541 sub foreach_drive {
2542     my ($conf, $func) = @_;
2543
2544     foreach my $ds (keys %$conf) {
2545         next if !valid_drivename($ds);
2546
2547         my $drive = parse_drive($ds, $conf->{$ds});
2548         next if !$drive;
2549
2550         &$func($ds, $drive);
2551     }
2552 }
2553
2554 sub foreach_volid {
2555     my ($conf, $func) = @_;
2556
2557     my $volhash = {};
2558
2559     my $test_volid = sub {
2560         my ($volid, $is_cdrom) = @_;
2561
2562         return if !$volid;
2563
2564         $volhash->{$volid} = $is_cdrom || 0;
2565     };
2566
2567     foreach_drive($conf, sub {
2568         my ($ds, $drive) = @_;
2569         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2570     });
2571
2572     foreach my $snapname (keys %{$conf->{snapshots}}) {
2573         my $snap = $conf->{snapshots}->{$snapname};
2574         &$test_volid($snap->{vmstate}, 0);
2575         foreach_drive($snap, sub {
2576             my ($ds, $drive) = @_;
2577             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2578         });
2579     }
2580
2581     foreach my $volid (keys %$volhash) {
2582         &$func($volid, $volhash->{$volid});
2583     }
2584 }
2585
2586 sub vga_conf_has_spice {
2587     my ($vga) = @_;
2588
2589     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2590
2591     return $1 || 1;
2592 }
2593
2594 sub config_to_command {
2595     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2596
2597     my $cmd = [];
2598     my $globalFlags = [];
2599     my $machineFlags = [];
2600     my $rtcFlags = [];
2601     my $cpuFlags = [];
2602     my $devices = [];
2603     my $pciaddr = '';
2604     my $bridges = {};
2605     my $kvmver = kvm_user_version();
2606     my $vernum = 0; # unknown
2607     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2608         $vernum = $1*1000000+$2*1000;
2609     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2610         $vernum = $1*1000000+$2*1000+$3;
2611     }
2612
2613     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2614
2615     my $have_ovz = -f '/proc/vz/vestat';
2616
2617     my $q35 = machine_type_is_q35($conf);
2618     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2619     my $machine_type = $forcemachine || $conf->{machine};
2620
2621     push @$cmd, '/usr/bin/kvm';
2622
2623     push @$cmd, '-id', $vmid;
2624
2625     my $use_virtio = 0;
2626
2627     my $qmpsocket = qmp_socket($vmid);
2628     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2629     push @$cmd, '-mon', "chardev=qmp,mode=control";
2630
2631     my $socket = vnc_socket($vmid);
2632     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2633
2634     push @$cmd, '-pidfile' , pidfile_name($vmid);
2635
2636     push @$cmd, '-daemonize';
2637
2638     if ($conf->{smbios1}) {
2639         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2640     }
2641
2642     if ($q35) {
2643         # the q35 chipset support native usb2, so we enable usb controller
2644         # by default for this machine type
2645         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2646     } else {
2647         $pciaddr = print_pci_addr("piix3", $bridges);
2648         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2649
2650         my $use_usb2 = 0;
2651         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2652             next if !$conf->{"usb$i"};
2653             $use_usb2 = 1;
2654         }
2655         # include usb device config
2656         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2657     }
2658
2659     my $vga = $conf->{vga};
2660
2661     my $qxlnum = vga_conf_has_spice($vga);
2662     $vga = 'qxl' if $qxlnum;
2663
2664     if (!$vga) {
2665         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2666                                 $conf->{ostype} eq 'win7' ||
2667                                 $conf->{ostype} eq 'w2k8')) {
2668             $vga = 'std';
2669         } else {
2670             $vga = 'cirrus';
2671         }
2672     }
2673
2674     # enable absolute mouse coordinates (needed by vnc)
2675     my $tablet;
2676     if (defined($conf->{tablet})) {
2677         $tablet = $conf->{tablet};
2678     } else {
2679         $tablet = $defaults->{tablet};
2680         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2681         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2682     }
2683
2684     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2685
2686     # host pci devices
2687     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2688         my $d = parse_hostpci($conf->{"hostpci$i"});
2689         next if !$d;
2690
2691         my $pcie = $d->{pcie};
2692         if($pcie){
2693             die "q35 machine model is not enabled" if !$q35;
2694             $pciaddr = print_pcie_addr("hostpci$i");
2695         }else{
2696             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2697         }
2698
2699         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2700         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2701         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2702         if ($xvga && $xvga ne '') {
2703             push @$cpuFlags, 'kvm=off';
2704             $vga = 'none';
2705         }
2706         $driver = "vfio-pci" if $xvga ne '';
2707         my $pcidevices = $d->{pciid};
2708         my $multifunction = 1 if @$pcidevices > 1;
2709
2710         my $j=0;
2711         foreach my $pcidevice (@$pcidevices) {
2712
2713             my $id = "hostpci$i";
2714             $id .= ".$j" if $multifunction;
2715             my $addr = $pciaddr;
2716             $addr .= ".$j" if $multifunction;
2717             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2718
2719             if($j == 0){
2720                 $devicestr .= "$rombar$xvga";
2721                 $devicestr .= ",multifunction=on" if $multifunction;
2722             }
2723
2724             push @$devices, '-device', $devicestr;
2725             $j++;
2726         }
2727     }
2728
2729     # usb devices
2730     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2731         my $d = parse_usb_device($conf->{"usb$i"});
2732         next if !$d;
2733         if ($d->{vendorid} && $d->{productid}) {
2734             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2735         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2736             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2737         } elsif ($d->{spice}) {
2738             # usb redir support for spice
2739             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2740             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2741         }
2742     }
2743
2744     # serial devices
2745     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2746         if (my $path = $conf->{"serial$i"}) {
2747             if ($path eq 'socket') {
2748                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2749                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2750                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2751             } else {
2752                 die "no such serial device\n" if ! -c $path;
2753                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2754                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2755             }
2756         }
2757     }
2758
2759     # parallel devices
2760     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2761         if (my $path = $conf->{"parallel$i"}) {
2762             die "no such parallel device\n" if ! -c $path;
2763             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2764             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2765             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2766         }
2767     }
2768
2769     my $vmname = $conf->{name} || "vm$vmid";
2770
2771     push @$cmd, '-name', $vmname;
2772
2773     my $sockets = 1;
2774     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2775     $sockets = $conf->{sockets} if  $conf->{sockets};
2776
2777     my $cores = $conf->{cores} || 1;
2778
2779     my $maxcpus = $sockets * $cores;
2780
2781     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2782
2783     my $allowed_vcpus = $cpuinfo->{cpus};
2784
2785     die "MAX $maxcpus vcpus allowed per VM on this node\n"
2786         if ($allowed_vcpus < $maxcpus);
2787
2788     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2789
2790     push @$cmd, '-nodefaults';
2791
2792     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2793
2794     my $bootindex_hash = {};
2795     my $i = 1;
2796     foreach my $o (split(//, $bootorder)) {
2797         $bootindex_hash->{$o} = $i*100;
2798         $i++;
2799     }
2800
2801     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2802
2803     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2804
2805     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2806
2807     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2808
2809     # time drift fix
2810     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2811
2812     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2813     my $useLocaltime = $conf->{localtime};
2814
2815     if (my $ost = $conf->{ostype}) {
2816         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2817
2818         if ($ost =~ m/^w/) { # windows
2819             $useLocaltime = 1 if !defined($conf->{localtime});
2820
2821             # use time drift fix when acpi is enabled
2822             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2823                 $tdf = 1 if !defined($conf->{tdf});
2824             }
2825         }
2826
2827         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2828             $ost eq 'wvista') {
2829             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2830             push @$cmd, '-no-hpet';
2831             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2832                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2833                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2834                 push @$cpuFlags , 'hv_time' if !$nokvm;
2835
2836             } else {
2837                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2838             }
2839         }
2840
2841         if ($ost eq 'win7' || $ost eq 'win8') {
2842             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2843         }
2844     }
2845
2846     push @$rtcFlags, 'driftfix=slew' if $tdf;
2847
2848     if ($nokvm) {
2849         push @$machineFlags, 'accel=tcg';
2850     } else {
2851         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2852     }
2853
2854     if ($machine_type) {
2855         push @$machineFlags, "type=${machine_type}";
2856     }
2857
2858     if ($conf->{startdate}) {
2859         push @$rtcFlags, "base=$conf->{startdate}";
2860     } elsif ($useLocaltime) {
2861         push @$rtcFlags, 'base=localtime';
2862     }
2863
2864     my $cpu = $nokvm ? "qemu64" : "kvm64";
2865     $cpu = $conf->{cpu} if $conf->{cpu};
2866
2867     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2868
2869     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2870
2871     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2872
2873     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2874
2875     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2876
2877         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2878         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2879     }
2880
2881     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2882
2883     push @$cmd, '-cpu', "$cpu,enforce";
2884
2885     my $memory = $conf->{memory} || $defaults->{memory};
2886     my $static_memory = 0;
2887     my $dimm_memory = 0;
2888
2889     if ($hotplug_features->{memory}) {
2890         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2891         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2892         $static_memory = $STATICMEM;
2893         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2894         $dimm_memory = $memory - $static_memory;
2895         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2896
2897     } else {
2898
2899         $static_memory = $memory;
2900         push @$cmd, '-m', $static_memory;
2901     }
2902
2903     if ($conf->{numa}) {
2904
2905         my $numa_totalmemory = undef;
2906         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2907             next if !$conf->{"numa$i"};
2908             my $numa = parse_numa($conf->{"numa$i"});
2909             next if !$numa;
2910             # memory
2911             die "missing numa node$i memory value\n" if !$numa->{memory};
2912             my $numa_memory = $numa->{memory};
2913             $numa_totalmemory += $numa_memory;
2914             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2915
2916             # cpus
2917             my $cpus_start = $numa->{cpus}->{start};
2918             die "missing numa node$i cpus\n" if !defined($cpus_start);
2919             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2920             my $cpus = $cpus_start;
2921             if (defined($cpus_end)) {
2922                 $cpus .= "-$cpus_end";
2923                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2924             }
2925
2926             # hostnodes
2927             my $hostnodes_start = $numa->{hostnodes}->{start};
2928             if (defined($hostnodes_start)) {
2929                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2930                 my $hostnodes = $hostnodes_start;
2931                 if (defined($hostnodes_end)) {
2932                     $hostnodes .= "-$hostnodes_end";
2933                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2934                 }
2935
2936                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2937                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2938                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2939                 }
2940
2941                 # policy
2942                 my $policy = $numa->{policy};
2943                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2944                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2945             }
2946
2947             push @$cmd, '-object', $numa_object;
2948             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2949         }
2950
2951         die "total memory for NUMA nodes must be equal to vm static memory\n"
2952             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2953
2954         #if no custom tology, we split memory and cores across numa nodes
2955         if(!$numa_totalmemory) {
2956
2957             my $numa_memory = ($static_memory / $sockets) . "M";
2958
2959             for (my $i = 0; $i < $sockets; $i++)  {
2960
2961                 my $cpustart = ($cores * $i);
2962                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2963                 my $cpus = $cpustart;
2964                 $cpus .= "-$cpuend" if $cpuend;
2965
2966                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2967                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2968             }
2969         }
2970     }
2971
2972     if ($hotplug_features->{memory}) {
2973         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
2974             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
2975             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
2976             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
2977
2978             #if dimm_memory is not aligned to dimm map
2979             if($current_size > $memory) {
2980                  $conf->{memory} = $current_size;
2981                  update_config_nolock($vmid, $conf, 1);
2982             }
2983         });
2984     }
2985
2986     push @$cmd, '-S' if $conf->{freeze};
2987
2988     # set keyboard layout
2989     my $kb = $conf->{keyboard} || $defaults->{keyboard};
2990     push @$cmd, '-k', $kb if $kb;
2991
2992     # enable sound
2993     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
2994     #push @$cmd, '-soundhw', 'es1370';
2995     #push @$cmd, '-soundhw', $soundhw if $soundhw;
2996
2997     if($conf->{agent}) {
2998         my $qgasocket = qmp_socket($vmid, 1);
2999         my $pciaddr = print_pci_addr("qga0", $bridges);
3000         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3001         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3002         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3003     }
3004
3005     my $spice_port;
3006
3007     if ($qxlnum) {
3008         if ($qxlnum > 1) {
3009             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3010                 for(my $i = 1; $i < $qxlnum; $i++){
3011                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3012                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3013                 }
3014             } else {
3015                 # assume other OS works like Linux
3016                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3017                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3018             }
3019         }
3020
3021         my $pciaddr = print_pci_addr("spice", $bridges);
3022
3023         $spice_port = PVE::Tools::next_spice_port();
3024
3025         push @$devices, '-spice', "tls-port=${spice_port},addr=127.0.0.1,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3026
3027         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3028         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3029         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3030     }
3031
3032     # enable balloon by default, unless explicitly disabled
3033     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3034         $pciaddr = print_pci_addr("balloon0", $bridges);
3035         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3036     }
3037
3038     if ($conf->{watchdog}) {
3039         my $wdopts = parse_watchdog($conf->{watchdog});
3040         $pciaddr = print_pci_addr("watchdog", $bridges);
3041         my $watchdog = $wdopts->{model} || 'i6300esb';
3042         push @$devices, '-device', "$watchdog$pciaddr";
3043         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3044     }
3045
3046     my $vollist = [];
3047     my $scsicontroller = {};
3048     my $ahcicontroller = {};
3049     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3050
3051     # Add iscsi initiator name if available
3052     if (my $initiator = get_initiator_name()) {
3053         push @$devices, '-iscsi', "initiator-name=$initiator";
3054     }
3055
3056     foreach_drive($conf, sub {
3057         my ($ds, $drive) = @_;
3058
3059         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3060             push @$vollist, $drive->{file};
3061         }
3062
3063         $use_virtio = 1 if $ds =~ m/^virtio/;
3064
3065         if (drive_is_cdrom ($drive)) {
3066             if ($bootindex_hash->{d}) {
3067                 $drive->{bootindex} = $bootindex_hash->{d};
3068                 $bootindex_hash->{d} += 1;
3069             }
3070         } else {
3071             if ($bootindex_hash->{c}) {
3072                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3073                 $bootindex_hash->{c} += 1;
3074             }
3075         }
3076
3077         if($drive->{interface} eq 'virtio'){
3078            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3079         }
3080
3081         if ($drive->{interface} eq 'scsi') {
3082
3083             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3084
3085             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3086             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3087
3088             my $iothread = '';
3089             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3090                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3091                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3092             }
3093
3094             my $queues = '';
3095             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3096                 $queues = ",num_queues=$drive->{queues}";
3097             }
3098
3099             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3100             $scsicontroller->{$controller}=1;
3101         }
3102
3103         if ($drive->{interface} eq 'sata') {
3104            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3105            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3106            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3107            $ahcicontroller->{$controller}=1;
3108         }
3109
3110         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3111         push @$devices, '-drive',$drive_cmd;
3112         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3113     });
3114
3115     for (my $i = 0; $i < $MAX_NETS; $i++) {
3116          next if !$conf->{"net$i"};
3117          my $d = parse_net($conf->{"net$i"});
3118          next if !$d;
3119
3120          $use_virtio = 1 if $d->{model} eq 'virtio';
3121
3122          if ($bootindex_hash->{n}) {
3123             $d->{bootindex} = $bootindex_hash->{n};
3124             $bootindex_hash->{n} += 1;
3125          }
3126
3127          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3128          push @$devices, '-netdev', $netdevfull;
3129
3130          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3131          push @$devices, '-device', $netdevicefull;
3132     }
3133
3134     if (!$q35) {
3135         # add pci bridges
3136         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3137            $bridges->{1} = 1;
3138            $bridges->{2} = 1;
3139         }
3140
3141         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3142
3143         while (my ($k, $v) = each %$bridges) {
3144             $pciaddr = print_pci_addr("pci.$k");
3145             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3146         }
3147     }
3148
3149     # hack: virtio with fairsched is unreliable, so we do not use fairsched
3150     # when the VM uses virtio devices.
3151     if (!$use_virtio && $have_ovz) {
3152
3153         my $cpuunits = defined($conf->{cpuunits}) ?
3154             $conf->{cpuunits} : $defaults->{cpuunits};
3155
3156         push @$cmd, '-cpuunits', $cpuunits if $cpuunits;
3157
3158         # fixme: cpulimit is currently ignored
3159         #push @$cmd, '-cpulimit', $conf->{cpulimit} if $conf->{cpulimit};
3160     }
3161
3162     # add custom args
3163     if ($conf->{args}) {
3164         my $aa = PVE::Tools::split_args($conf->{args});
3165         push @$cmd, @$aa;
3166     }
3167
3168     push @$cmd, @$devices;
3169     push @$cmd, '-rtc', join(',', @$rtcFlags)
3170         if scalar(@$rtcFlags);
3171     push @$cmd, '-machine', join(',', @$machineFlags)
3172         if scalar(@$machineFlags);
3173     push @$cmd, '-global', join(',', @$globalFlags)
3174         if scalar(@$globalFlags);
3175
3176     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3177 }
3178
3179 sub vnc_socket {
3180     my ($vmid) = @_;
3181     return "${var_run_tmpdir}/$vmid.vnc";
3182 }
3183
3184 sub spice_port {
3185     my ($vmid) = @_;
3186
3187     my $res = vm_mon_cmd($vmid, 'query-spice');
3188
3189     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3190 }
3191
3192 sub qmp_socket {
3193     my ($vmid, $qga) = @_;
3194     my $sockettype = $qga ? 'qga' : 'qmp';
3195     return "${var_run_tmpdir}/$vmid.$sockettype";
3196 }
3197
3198 sub pidfile_name {
3199     my ($vmid) = @_;
3200     return "${var_run_tmpdir}/$vmid.pid";
3201 }
3202
3203 sub vm_devices_list {
3204     my ($vmid) = @_;
3205
3206     my $res = vm_mon_cmd($vmid, 'query-pci');
3207     my $devices = {};
3208     foreach my $pcibus (@$res) {
3209         foreach my $device (@{$pcibus->{devices}}) {
3210             next if !$device->{'qdev_id'};
3211             if ($device->{'pci_bridge'}) {
3212                 $devices->{$device->{'qdev_id'}} = 1;
3213                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3214                     next if !$bridge_device->{'qdev_id'};
3215                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3216                     $devices->{$device->{'qdev_id'}}++;
3217                 }
3218             } else {
3219                 $devices->{$device->{'qdev_id'}} = 1;
3220             }
3221         }
3222     }
3223
3224     my $resblock = vm_mon_cmd($vmid, 'query-block');
3225     foreach my $block (@$resblock) {
3226         if($block->{device} =~ m/^drive-(\S+)/){
3227                 $devices->{$1} = 1;
3228         }
3229     }
3230
3231     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3232     foreach my $mice (@$resmice) {
3233         if ($mice->{name} eq 'QEMU HID Tablet') {
3234             $devices->{tablet} = 1;
3235             last;
3236         }
3237     }
3238
3239     return $devices;
3240 }
3241
3242 sub vm_deviceplug {
3243     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3244
3245     my $q35 = machine_type_is_q35($conf);
3246
3247     my $devices_list = vm_devices_list($vmid);
3248     return 1 if defined($devices_list->{$deviceid});
3249
3250     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3251
3252     if ($deviceid eq 'tablet') {
3253
3254         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3255
3256     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3257
3258         qemu_iothread_add($vmid, $deviceid, $device);
3259
3260         qemu_driveadd($storecfg, $vmid, $device);
3261         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3262
3263         qemu_deviceadd($vmid, $devicefull);
3264         eval { qemu_deviceaddverify($vmid, $deviceid); };
3265         if (my $err = $@) {
3266             eval { qemu_drivedel($vmid, $deviceid); };
3267             warn $@ if $@;
3268             die $err;
3269         }
3270
3271     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3272
3273
3274         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3275         my $pciaddr = print_pci_addr($deviceid);
3276         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3277
3278         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3279
3280         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3281             qemu_iothread_add($vmid, $deviceid, $device);
3282             $devicefull .= ",iothread=iothread-$deviceid";
3283         }
3284
3285         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3286             $devicefull .= ",num_queues=$device->{queues}";
3287         }
3288
3289         qemu_deviceadd($vmid, $devicefull);
3290         qemu_deviceaddverify($vmid, $deviceid);
3291
3292     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3293
3294         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3295         qemu_driveadd($storecfg, $vmid, $device);
3296
3297         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3298         eval { qemu_deviceadd($vmid, $devicefull); };
3299         if (my $err = $@) {
3300             eval { qemu_drivedel($vmid, $deviceid); };
3301             warn $@ if $@;
3302             die $err;
3303         }
3304
3305     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3306
3307         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3308         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3309         qemu_deviceadd($vmid, $netdevicefull);
3310         eval { qemu_deviceaddverify($vmid, $deviceid); };
3311         if (my $err = $@) {
3312             eval { qemu_netdevdel($vmid, $deviceid); };
3313             warn $@ if $@;
3314             die $err;
3315         }
3316
3317     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3318
3319         my $bridgeid = $2;
3320         my $pciaddr = print_pci_addr($deviceid);
3321         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3322
3323         qemu_deviceadd($vmid, $devicefull);
3324         qemu_deviceaddverify($vmid, $deviceid);
3325
3326     } else {
3327         die "can't hotplug device '$deviceid'\n";
3328     }
3329
3330     return 1;
3331 }
3332
3333 # fixme: this should raise exceptions on error!
3334 sub vm_deviceunplug {
3335     my ($vmid, $conf, $deviceid) = @_;
3336
3337     my $devices_list = vm_devices_list($vmid);
3338     return 1 if !defined($devices_list->{$deviceid});
3339
3340     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3341
3342     if ($deviceid eq 'tablet') {
3343
3344         qemu_devicedel($vmid, $deviceid);
3345
3346     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3347
3348         qemu_devicedel($vmid, $deviceid);
3349         qemu_devicedelverify($vmid, $deviceid);
3350         qemu_drivedel($vmid, $deviceid);
3351         qemu_iothread_del($conf, $vmid, $deviceid);
3352
3353     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3354
3355         qemu_devicedel($vmid, $deviceid);
3356         qemu_devicedelverify($vmid, $deviceid);
3357         qemu_iothread_del($conf, $vmid, $deviceid);
3358
3359     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3360
3361         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3362         my $device = parse_drive($deviceid, $conf->{$deviceid});
3363         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3364
3365         qemu_devicedel($vmid, $deviceid);
3366         qemu_drivedel($vmid, $deviceid);
3367         qemu_deletescsihw($conf, $vmid, $deviceid);
3368
3369     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3370
3371         qemu_devicedel($vmid, $deviceid);
3372         qemu_devicedelverify($vmid, $deviceid);
3373         qemu_netdevdel($vmid, $deviceid);
3374
3375     } else {
3376         die "can't unplug device '$deviceid'\n";
3377     }
3378
3379     return 1;
3380 }
3381
3382 sub qemu_deviceadd {
3383     my ($vmid, $devicefull) = @_;
3384
3385     $devicefull = "driver=".$devicefull;
3386     my %options =  split(/[=,]/, $devicefull);
3387
3388     vm_mon_cmd($vmid, "device_add" , %options);
3389 }
3390
3391 sub qemu_devicedel {
3392     my ($vmid, $deviceid) = @_;
3393
3394     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3395 }
3396
3397 sub qemu_iothread_add {
3398     my($vmid, $deviceid, $device) = @_;
3399
3400     if ($device->{iothread}) {
3401         my $iothreads = vm_iothreads_list($vmid);
3402         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3403     }
3404 }
3405
3406 sub qemu_iothread_del {
3407     my($conf, $vmid, $deviceid) = @_;
3408
3409     my $device = parse_drive($deviceid, $conf->{$deviceid});
3410     if ($device->{iothread}) {
3411         my $iothreads = vm_iothreads_list($vmid);
3412         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3413     }
3414 }
3415
3416 sub qemu_objectadd {
3417     my($vmid, $objectid, $qomtype) = @_;
3418
3419     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3420
3421     return 1;
3422 }
3423
3424 sub qemu_objectdel {
3425     my($vmid, $objectid) = @_;
3426
3427     vm_mon_cmd($vmid, "object-del", id => $objectid);
3428
3429     return 1;
3430 }
3431
3432 sub qemu_driveadd {
3433     my ($storecfg, $vmid, $device) = @_;
3434
3435     my $drive = print_drive_full($storecfg, $vmid, $device);
3436     $drive =~ s/\\/\\\\/g;
3437     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3438
3439     # If the command succeeds qemu prints: "OK"
3440     return 1 if $ret =~ m/OK/s;
3441
3442     die "adding drive failed: $ret\n";
3443 }
3444
3445 sub qemu_drivedel {
3446     my($vmid, $deviceid) = @_;
3447
3448     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3449     $ret =~ s/^\s+//;
3450
3451     return 1 if $ret eq "";
3452
3453     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3454     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3455
3456     die "deleting drive $deviceid failed : $ret\n";
3457 }
3458
3459 sub qemu_deviceaddverify {
3460     my ($vmid, $deviceid) = @_;
3461
3462     for (my $i = 0; $i <= 5; $i++) {
3463          my $devices_list = vm_devices_list($vmid);
3464          return 1 if defined($devices_list->{$deviceid});
3465          sleep 1;
3466     }
3467
3468     die "error on hotplug device '$deviceid'\n";
3469 }
3470
3471
3472 sub qemu_devicedelverify {
3473     my ($vmid, $deviceid) = @_;
3474
3475     # need to verify that the device is correctly removed as device_del
3476     # is async and empty return is not reliable
3477
3478     for (my $i = 0; $i <= 5; $i++) {
3479          my $devices_list = vm_devices_list($vmid);
3480          return 1 if !defined($devices_list->{$deviceid});
3481          sleep 1;
3482     }
3483
3484     die "error on hot-unplugging device '$deviceid'\n";
3485 }
3486
3487 sub qemu_findorcreatescsihw {
3488     my ($storecfg, $conf, $vmid, $device) = @_;
3489
3490     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3491
3492     my $scsihwid="$controller_prefix$controller";
3493     my $devices_list = vm_devices_list($vmid);
3494
3495     if(!defined($devices_list->{$scsihwid})) {
3496         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3497     }
3498
3499     return 1;
3500 }
3501
3502 sub qemu_deletescsihw {
3503     my ($conf, $vmid, $opt) = @_;
3504
3505     my $device = parse_drive($opt, $conf->{$opt});
3506
3507     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3508         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3509         return 1;
3510     }
3511
3512     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3513
3514     my $devices_list = vm_devices_list($vmid);
3515     foreach my $opt (keys %{$devices_list}) {
3516         if (PVE::QemuServer::valid_drivename($opt)) {
3517             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3518             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3519                 return 1;
3520             }
3521         }
3522     }
3523
3524     my $scsihwid="scsihw$controller";
3525
3526     vm_deviceunplug($vmid, $conf, $scsihwid);
3527
3528     return 1;
3529 }
3530
3531 sub qemu_add_pci_bridge {
3532     my ($storecfg, $conf, $vmid, $device) = @_;
3533
3534     my $bridges = {};
3535
3536     my $bridgeid;
3537
3538     print_pci_addr($device, $bridges);
3539
3540     while (my ($k, $v) = each %$bridges) {
3541         $bridgeid = $k;
3542     }
3543     return 1 if !defined($bridgeid) || $bridgeid < 1;
3544
3545     my $bridge = "pci.$bridgeid";
3546     my $devices_list = vm_devices_list($vmid);
3547
3548     if (!defined($devices_list->{$bridge})) {
3549         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3550     }
3551
3552     return 1;
3553 }
3554
3555 sub qemu_set_link_status {
3556     my ($vmid, $device, $up) = @_;
3557
3558     vm_mon_cmd($vmid, "set_link", name => $device,
3559                up => $up ? JSON::true : JSON::false);
3560 }
3561
3562 sub qemu_netdevadd {
3563     my ($vmid, $conf, $device, $deviceid) = @_;
3564
3565     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3566     my %options =  split(/[=,]/, $netdev);
3567
3568     vm_mon_cmd($vmid, "netdev_add",  %options);
3569     return 1;
3570 }
3571
3572 sub qemu_netdevdel {
3573     my ($vmid, $deviceid) = @_;
3574
3575     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3576 }
3577
3578 sub qemu_cpu_hotplug {
3579     my ($vmid, $conf, $vcpus) = @_;
3580
3581     my $sockets = 1;
3582     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3583     $sockets = $conf->{sockets} if  $conf->{sockets};
3584     my $cores = $conf->{cores} || 1;
3585     my $maxcpus = $sockets * $cores;
3586
3587     $vcpus = $maxcpus if !$vcpus;
3588
3589     die "you can't add more vcpus than maxcpus\n"
3590         if $vcpus > $maxcpus;
3591
3592     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3593     die "online cpu unplug is not yet possible\n"
3594         if $vcpus < $currentvcpus;
3595
3596     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3597     die "vcpus in running vm is different than configuration\n"
3598         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3599
3600     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3601         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3602     }
3603 }
3604
3605 sub qemu_memory_hotplug {
3606     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3607
3608     return $value if !check_running($vmid);
3609
3610     my $memory = $conf->{memory} || $defaults->{memory};
3611     $value = $defaults->{memory} if !$value;
3612     return $value if $value == $memory;
3613
3614     my $static_memory = $STATICMEM;
3615     my $dimm_memory = $memory - $static_memory;
3616
3617     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3618     die "memory unplug is not yet available" if $value < $memory;
3619     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3620
3621
3622     my $sockets = 1;
3623     $sockets = $conf->{sockets} if $conf->{sockets};
3624
3625     foreach_dimm($conf, $vmid, $value, $sockets, sub {
3626         my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3627
3628             return if $current_size <= $conf->{memory};
3629
3630             eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3631             if (my $err = $@) {
3632                 eval { qemu_objectdel($vmid, "mem-$name"); };
3633                 die $err;
3634             }
3635
3636             eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3637             if (my $err = $@) {
3638                 eval { qemu_objectdel($vmid, "mem-$name"); };
3639                 die $err;
3640             }
3641             #update conf after each succesful module hotplug
3642             $conf->{memory} = $current_size;
3643             update_config_nolock($vmid, $conf, 1);
3644     });
3645 }
3646
3647 sub qemu_block_set_io_throttle {
3648     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3649
3650     return if !check_running($vmid) ;
3651
3652     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3653
3654 }
3655
3656 # old code, only used to shutdown old VM after update
3657 sub __read_avail {
3658     my ($fh, $timeout) = @_;
3659
3660     my $sel = new IO::Select;
3661     $sel->add($fh);
3662
3663     my $res = '';
3664     my $buf;
3665
3666     my @ready;
3667     while (scalar (@ready = $sel->can_read($timeout))) {
3668         my $count;
3669         if ($count = $fh->sysread($buf, 8192)) {
3670             if ($buf =~ /^(.*)\(qemu\) $/s) {
3671                 $res .= $1;
3672                 last;
3673             } else {
3674                 $res .= $buf;
3675             }
3676         } else {
3677             if (!defined($count)) {
3678                 die "$!\n";
3679             }
3680             last;
3681         }
3682     }
3683
3684     die "monitor read timeout\n" if !scalar(@ready);
3685
3686     return $res;
3687 }
3688
3689 # old code, only used to shutdown old VM after update
3690 sub vm_monitor_command {
3691     my ($vmid, $cmdstr, $nocheck) = @_;
3692
3693     my $res;
3694
3695     eval {
3696         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3697
3698         my $sname = "${var_run_tmpdir}/$vmid.mon";
3699
3700         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3701             die "unable to connect to VM $vmid socket - $!\n";
3702
3703         my $timeout = 3;
3704
3705         # hack: migrate sometime blocks the monitor (when migrate_downtime
3706         # is set)
3707         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3708             $timeout = 60*60; # 1 hour
3709         }
3710
3711         # read banner;
3712         my $data = __read_avail($sock, $timeout);
3713
3714         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3715             die "got unexpected qemu monitor banner\n";
3716         }
3717
3718         my $sel = new IO::Select;
3719         $sel->add($sock);
3720
3721         if (!scalar(my @ready = $sel->can_write($timeout))) {
3722             die "monitor write error - timeout";
3723         }
3724
3725         my $fullcmd = "$cmdstr\r";
3726
3727         # syslog('info', "VM $vmid monitor command: $cmdstr");
3728
3729         my $b;
3730         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3731             die "monitor write error - $!";
3732         }
3733
3734         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3735
3736         $timeout = 20;
3737
3738         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3739             $timeout = 60*60; # 1 hour
3740         } elsif ($cmdstr =~ m/^(eject|change)/) {
3741             $timeout = 60; # note: cdrom mount command is slow
3742         }
3743         if ($res = __read_avail($sock, $timeout)) {
3744
3745             my @lines = split("\r?\n", $res);
3746
3747             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3748
3749             $res = join("\n", @lines);
3750             $res .= "\n";
3751         }
3752     };
3753
3754     my $err = $@;
3755
3756     if ($err) {
3757         syslog("err", "VM $vmid monitor command failed - $err");
3758         die $err;
3759     }
3760
3761     return $res;
3762 }
3763
3764 sub qemu_block_resize {
3765     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3766
3767     my $running = check_running($vmid);
3768
3769     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3770
3771     return if !$running;
3772
3773     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3774
3775 }
3776
3777 sub qemu_volume_snapshot {
3778     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3779
3780     my $running = check_running($vmid);
3781
3782     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3783         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3784     } else {
3785         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3786     }
3787 }
3788
3789 sub qemu_volume_snapshot_delete {
3790     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3791
3792     my $running = check_running($vmid);
3793
3794     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3795
3796     return if !$running;
3797
3798     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3799 }
3800
3801 sub set_migration_caps {
3802     my ($vmid) = @_;
3803
3804     my $cap_ref = [];
3805
3806     my $enabled_cap = {
3807         "auto-converge" => 1,
3808         "xbzrle" => 0,
3809         "x-rdma-pin-all" => 0,
3810         "zero-blocks" => 0,
3811     };
3812
3813     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3814
3815     for my $supported_capability (@$supported_capabilities) {
3816         push @$cap_ref, {
3817             capability => $supported_capability->{capability},
3818             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3819         };
3820     }
3821
3822     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3823 }
3824
3825 my $fast_plug_option = {
3826     'lock' => 1,
3827     'name' => 1,
3828     'onboot' => 1,
3829     'shares' => 1,
3830     'startup' => 1,
3831 };
3832
3833 # hotplug changes in [PENDING]
3834 # $selection hash can be used to only apply specified options, for
3835 # example: { cores => 1 } (only apply changed 'cores')
3836 # $errors ref is used to return error messages
3837 sub vmconfig_hotplug_pending {
3838     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3839
3840     my $defaults = load_defaults();
3841
3842     # commit values which do not have any impact on running VM first
3843     # Note: those option cannot raise errors, we we do not care about
3844     # $selection and always apply them.
3845
3846     my $add_error = sub {
3847         my ($opt, $msg) = @_;
3848         $errors->{$opt} = "hotplug problem - $msg";
3849     };
3850
3851     my $changes = 0;
3852     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3853         if ($fast_plug_option->{$opt}) {
3854             $conf->{$opt} = $conf->{pending}->{$opt};
3855             delete $conf->{pending}->{$opt};
3856             $changes = 1;
3857         }
3858     }
3859
3860     if ($changes) {
3861         update_config_nolock($vmid, $conf, 1);
3862         $conf = load_config($vmid); # update/reload
3863     }
3864
3865     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3866
3867     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3868     foreach my $opt (@delete) {
3869         next if $selection && !$selection->{$opt};
3870         eval {
3871             if ($opt eq 'hotplug') {
3872                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3873             } elsif ($opt eq 'tablet') {
3874                 die "skip\n" if !$hotplug_features->{usb};
3875                 if ($defaults->{tablet}) {
3876                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3877                 } else {
3878                     vm_deviceunplug($vmid, $conf, $opt);
3879                 }
3880             } elsif ($opt eq 'vcpus') {
3881                 die "skip\n" if !$hotplug_features->{cpu};
3882                 qemu_cpu_hotplug($vmid, $conf, undef);
3883             } elsif ($opt eq 'balloon') {
3884                 # enable balloon device is not hotpluggable
3885                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3886             } elsif ($fast_plug_option->{$opt}) {
3887                 # do nothing
3888             } elsif ($opt =~ m/^net(\d+)$/) {
3889                 die "skip\n" if !$hotplug_features->{network};
3890                 vm_deviceunplug($vmid, $conf, $opt);
3891             } elsif (valid_drivename($opt)) {
3892                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3893                 vm_deviceunplug($vmid, $conf, $opt);
3894                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3895             } elsif ($opt =~ m/^memory$/) {
3896                 die "skip\n" if !$hotplug_features->{memory};
3897                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3898             } else {
3899                 die "skip\n";
3900             }
3901         };
3902         if (my $err = $@) {
3903             &$add_error($opt, $err) if $err ne "skip\n";
3904         } else {
3905             # save new config if hotplug was successful
3906             delete $conf->{$opt};
3907             vmconfig_undelete_pending_option($conf, $opt);
3908             update_config_nolock($vmid, $conf, 1);
3909             $conf = load_config($vmid); # update/reload
3910         }
3911     }
3912
3913     foreach my $opt (keys %{$conf->{pending}}) {
3914         next if $selection && !$selection->{$opt};
3915         my $value = $conf->{pending}->{$opt};
3916         eval {
3917             if ($opt eq 'hotplug') {
3918                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3919             } elsif ($opt eq 'tablet') {
3920                 die "skip\n" if !$hotplug_features->{usb};
3921                 if ($value == 1) {
3922                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3923                 } elsif ($value == 0) {
3924                     vm_deviceunplug($vmid, $conf, $opt);
3925                 }
3926             } elsif ($opt eq 'vcpus') {
3927                 die "skip\n" if !$hotplug_features->{cpu};
3928                 qemu_cpu_hotplug($vmid, $conf, $value);
3929             } elsif ($opt eq 'balloon') {
3930                 # enable/disable balloning device is not hotpluggable
3931                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3932                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3933                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3934
3935                 # allow manual ballooning if shares is set to zero
3936                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3937                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3938                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3939                 }
3940             } elsif ($opt =~ m/^net(\d+)$/) {
3941                 # some changes can be done without hotplug
3942                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3943                                     $vmid, $opt, $value);
3944             } elsif (valid_drivename($opt)) {
3945                 # some changes can be done without hotplug
3946                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3947                                      $vmid, $opt, $value, 1);
3948             } elsif ($opt =~ m/^memory$/) { #dimms
3949                 die "skip\n" if !$hotplug_features->{memory};
3950                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3951             } else {
3952                 die "skip\n";  # skip non-hot-pluggable options
3953             }
3954         };
3955         if (my $err = $@) {
3956             &$add_error($opt, $err) if $err ne "skip\n";
3957         } else {
3958             # save new config if hotplug was successful
3959             $conf->{$opt} = $value;
3960             delete $conf->{pending}->{$opt};
3961             update_config_nolock($vmid, $conf, 1);
3962             $conf = load_config($vmid); # update/reload
3963         }
3964     }
3965 }
3966
3967 sub vmconfig_apply_pending {
3968     my ($vmid, $conf, $storecfg) = @_;
3969
3970     # cold plug
3971
3972     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3973     foreach my $opt (@delete) { # delete
3974         die "internal error" if $opt =~ m/^unused/;
3975         $conf = load_config($vmid); # update/reload
3976         if (!defined($conf->{$opt})) {
3977             vmconfig_undelete_pending_option($conf, $opt);
3978             update_config_nolock($vmid, $conf, 1);
3979         } elsif (valid_drivename($opt)) {
3980             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3981             vmconfig_undelete_pending_option($conf, $opt);
3982             delete $conf->{$opt};
3983             update_config_nolock($vmid, $conf, 1);
3984         } else {
3985             vmconfig_undelete_pending_option($conf, $opt);
3986             delete $conf->{$opt};
3987             update_config_nolock($vmid, $conf, 1);
3988         }
3989     }
3990
3991     $conf = load_config($vmid); # update/reload
3992
3993     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3994         $conf = load_config($vmid); # update/reload
3995
3996         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
3997             # skip if nothing changed
3998         } elsif (valid_drivename($opt)) {
3999             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4000                 if defined($conf->{$opt});
4001             $conf->{$opt} = $conf->{pending}->{$opt};
4002         } else {
4003             $conf->{$opt} = $conf->{pending}->{$opt};
4004         }
4005
4006         delete $conf->{pending}->{$opt};
4007         update_config_nolock($vmid, $conf, 1);
4008     }
4009 }
4010
4011 my $safe_num_ne = sub {
4012     my ($a, $b) = @_;
4013
4014     return 0 if !defined($a) && !defined($b);
4015     return 1 if !defined($a);
4016     return 1 if !defined($b);
4017
4018     return $a != $b;
4019 };
4020
4021 my $safe_string_ne = sub {
4022     my ($a, $b) = @_;
4023
4024     return 0 if !defined($a) && !defined($b);
4025     return 1 if !defined($a);
4026     return 1 if !defined($b);
4027
4028     return $a ne $b;
4029 };
4030
4031 sub vmconfig_update_net {
4032     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4033
4034     my $newnet = parse_net($value);
4035
4036     if ($conf->{$opt}) {
4037         my $oldnet = parse_net($conf->{$opt});
4038
4039         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4040             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4041             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4042             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4043
4044             # for non online change, we try to hot-unplug
4045             die "skip\n" if !$hotplug;
4046             vm_deviceunplug($vmid, $conf, $opt);
4047         } else {
4048
4049             die "internal error" if $opt !~ m/net(\d+)/;
4050             my $iface = "tap${vmid}i$1";
4051
4052             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4053                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4054             }
4055
4056             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4057                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4058                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4059                 PVE::Network::tap_unplug($iface);
4060                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4061             }
4062
4063             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4064                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4065             }
4066
4067             return 1;
4068         }
4069     }
4070
4071     if ($hotplug) {
4072         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4073     } else {
4074         die "skip\n";
4075     }
4076 }
4077
4078 sub vmconfig_update_disk {
4079     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4080
4081     # fixme: do we need force?
4082
4083     my $drive = parse_drive($opt, $value);
4084
4085     if ($conf->{$opt}) {
4086
4087         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4088
4089             my $media = $drive->{media} || 'disk';
4090             my $oldmedia = $old_drive->{media} || 'disk';
4091             die "unable to change media type\n" if $media ne $oldmedia;
4092
4093             if (!drive_is_cdrom($old_drive)) {
4094
4095                 if ($drive->{file} ne $old_drive->{file}) {
4096
4097                     die "skip\n" if !$hotplug;
4098
4099                     # unplug and register as unused
4100                     vm_deviceunplug($vmid, $conf, $opt);
4101                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4102
4103                 } else {
4104                     # update existing disk
4105
4106                     # skip non hotpluggable value
4107                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4108                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4109                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4110                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4111                         die "skip\n";
4112                     }
4113
4114                     # apply throttle
4115                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4116                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4117                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4118                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4119                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4120                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4121                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4122                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4123                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4124                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4125                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4126                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4127
4128                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4129                                                    ($drive->{mbps} || 0)*1024*1024,
4130                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4131                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4132                                                    $drive->{iops} || 0,
4133                                                    $drive->{iops_rd} || 0,
4134                                                    $drive->{iops_wr} || 0,
4135                                                    ($drive->{mbps_max} || 0)*1024*1024,
4136                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4137                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4138                                                    $drive->{iops_max} || 0,
4139                                                    $drive->{iops_rd_max} || 0,
4140                                                    $drive->{iops_wr_max} || 0);
4141
4142                     }
4143
4144                     return 1;
4145                 }
4146
4147             } else { # cdrom
4148
4149                 if ($drive->{file} eq 'none') {
4150                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4151                 } else {
4152                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4153                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4154                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4155                 }
4156
4157                 return 1;
4158             }
4159         }
4160     }
4161
4162     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4163     # hotplug new disks
4164     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4165 }
4166
4167 sub vm_start {
4168     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4169
4170     lock_config($vmid, sub {
4171         my $conf = load_config($vmid, $migratedfrom);
4172
4173         die "you can't start a vm if it's a template\n" if is_template($conf);
4174
4175         check_lock($conf) if !$skiplock;
4176
4177         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4178
4179         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4180             vmconfig_apply_pending($vmid, $conf, $storecfg);
4181             $conf = load_config($vmid); # update/reload
4182         }
4183
4184         my $defaults = load_defaults();
4185
4186         # set environment variable useful inside network script
4187         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4188
4189         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4190
4191         my $migrate_port = 0;
4192         my $migrate_uri;
4193         if ($statefile) {
4194             if ($statefile eq 'tcp') {
4195                 my $localip = "localhost";
4196                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4197                 if ($datacenterconf->{migration_unsecure}) {
4198                         my $nodename = PVE::INotify::nodename();
4199                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4200                 }
4201                 $migrate_port = PVE::Tools::next_migrate_port();
4202                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4203                 push @$cmd, '-incoming', $migrate_uri;
4204                 push @$cmd, '-S';
4205             } else {
4206                 push @$cmd, '-loadstate', $statefile;
4207             }
4208         } elsif ($paused) {
4209             push @$cmd, '-S';
4210         }
4211
4212         # host pci devices
4213         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4214           my $d = parse_hostpci($conf->{"hostpci$i"});
4215           next if !$d;
4216           my $pcidevices = $d->{pciid};
4217           foreach my $pcidevice (@$pcidevices) {
4218                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4219
4220                 my $info = pci_device_info("0000:$pciid");
4221                 die "IOMMU not present\n" if !check_iommu_support();
4222                 die "no pci device info for device '$pciid'\n" if !$info;
4223
4224                 if ($d->{driver} && $d->{driver} eq "vfio") {
4225                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4226                 } else {
4227                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
4228                 }
4229
4230                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4231           }
4232         }
4233
4234         PVE::Storage::activate_volumes($storecfg, $vollist);
4235
4236         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4237                     umask => 0077); };
4238         my $err = $@;
4239         die "start failed: $err" if $err;
4240
4241         print "migration listens on $migrate_uri\n" if $migrate_uri;
4242
4243         if ($statefile && $statefile ne 'tcp')  {
4244             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4245             warn $@ if $@;
4246         }
4247
4248         if ($migratedfrom) {
4249
4250             eval {
4251                 set_migration_caps($vmid);
4252             };
4253             warn $@ if $@;
4254
4255             if ($spice_port) {
4256                 print "spice listens on port $spice_port\n";
4257                 if ($spice_ticket) {
4258                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4259                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4260                 }
4261             }
4262
4263         } else {
4264
4265             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4266                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4267                     if $conf->{balloon};
4268             }
4269
4270             foreach my $opt (keys %$conf) {
4271                 next if $opt !~  m/^net\d+$/;
4272                 my $nicconf = parse_net($conf->{$opt});
4273                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4274             }
4275         }
4276
4277         vm_mon_cmd_nocheck($vmid, 'qom-set',
4278                     path => "machine/peripheral/balloon0",
4279                     property => "guest-stats-polling-interval",
4280                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4281
4282     });
4283 }
4284
4285 sub vm_mon_cmd {
4286     my ($vmid, $execute, %params) = @_;
4287
4288     my $cmd = { execute => $execute, arguments => \%params };
4289     vm_qmp_command($vmid, $cmd);
4290 }
4291
4292 sub vm_mon_cmd_nocheck {
4293     my ($vmid, $execute, %params) = @_;
4294
4295     my $cmd = { execute => $execute, arguments => \%params };
4296     vm_qmp_command($vmid, $cmd, 1);
4297 }
4298
4299 sub vm_qmp_command {
4300     my ($vmid, $cmd, $nocheck) = @_;
4301
4302     my $res;
4303
4304     my $timeout;
4305     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4306         $timeout = $cmd->{arguments}->{timeout};
4307         delete $cmd->{arguments}->{timeout};
4308     }
4309
4310     eval {
4311         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4312         my $sname = qmp_socket($vmid);
4313         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4314             my $qmpclient = PVE::QMPClient->new();
4315
4316             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4317         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4318             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4319                 if scalar(%{$cmd->{arguments}});
4320             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4321         } else {
4322             die "unable to open monitor socket\n";
4323         }
4324     };
4325     if (my $err = $@) {
4326         syslog("err", "VM $vmid qmp command failed - $err");
4327         die $err;
4328     }
4329
4330     return $res;
4331 }
4332
4333 sub vm_human_monitor_command {
4334     my ($vmid, $cmdline) = @_;
4335
4336     my $res;
4337
4338     my $cmd = {
4339         execute => 'human-monitor-command',
4340         arguments => { 'command-line' => $cmdline},
4341     };
4342
4343     return vm_qmp_command($vmid, $cmd);
4344 }
4345
4346 sub vm_commandline {
4347     my ($storecfg, $vmid) = @_;
4348
4349     my $conf = load_config($vmid);
4350
4351     my $defaults = load_defaults();
4352
4353     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4354
4355     return join(' ', @$cmd);
4356 }
4357
4358 sub vm_reset {
4359     my ($vmid, $skiplock) = @_;
4360
4361     lock_config($vmid, sub {
4362
4363         my $conf = load_config($vmid);
4364
4365         check_lock($conf) if !$skiplock;
4366
4367         vm_mon_cmd($vmid, "system_reset");
4368     });
4369 }
4370
4371 sub get_vm_volumes {
4372     my ($conf) = @_;
4373
4374     my $vollist = [];
4375     foreach_volid($conf, sub {
4376         my ($volid, $is_cdrom) = @_;
4377
4378         return if $volid =~ m|^/|;
4379
4380         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4381         return if !$sid;
4382
4383         push @$vollist, $volid;
4384     });
4385
4386     return $vollist;
4387 }
4388
4389 sub vm_stop_cleanup {
4390     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4391
4392     eval {
4393         fairsched_rmnod($vmid); # try to destroy group
4394
4395         if (!$keepActive) {
4396             my $vollist = get_vm_volumes($conf);
4397             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4398         }
4399
4400         foreach my $ext (qw(mon qmp pid vnc qga)) {
4401             unlink "/var/run/qemu-server/${vmid}.$ext";
4402         }
4403
4404         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4405     };
4406     warn $@ if $@; # avoid errors - just warn
4407 }
4408
4409 # Note: use $nockeck to skip tests if VM configuration file exists.
4410 # We need that when migration VMs to other nodes (files already moved)
4411 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4412 sub vm_stop {
4413     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4414
4415     $force = 1 if !defined($force) && !$shutdown;
4416
4417     if ($migratedfrom){
4418         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4419         kill 15, $pid if $pid;
4420         my $conf = load_config($vmid, $migratedfrom);
4421         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4422         return;
4423     }
4424
4425     lock_config($vmid, sub {
4426
4427         my $pid = check_running($vmid, $nocheck);
4428         return if !$pid;
4429
4430         my $conf;
4431         if (!$nocheck) {
4432             $conf = load_config($vmid);
4433             check_lock($conf) if !$skiplock;
4434             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4435                 my $opts = parse_startup($conf->{startup});
4436                 $timeout = $opts->{down} if $opts->{down};
4437             }
4438         }
4439
4440         $timeout = 60 if !defined($timeout);
4441
4442         eval {
4443             if ($shutdown) {
4444                 if (defined($conf) && $conf->{agent}) {
4445                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4446                 } else {
4447                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4448                 }
4449             } else {
4450                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4451             }
4452         };
4453         my $err = $@;
4454
4455         if (!$err) {
4456             my $count = 0;
4457             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4458                 $count++;
4459                 sleep 1;
4460             }
4461
4462             if ($count >= $timeout) {
4463                 if ($force) {
4464                     warn "VM still running - terminating now with SIGTERM\n";
4465                     kill 15, $pid;
4466                 } else {
4467                     die "VM quit/powerdown failed - got timeout\n";
4468                 }
4469             } else {
4470                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4471                 return;
4472             }
4473         } else {
4474             if ($force) {
4475                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4476                 kill 15, $pid;
4477             } else {
4478                 die "VM quit/powerdown failed\n";
4479             }
4480         }
4481
4482         # wait again
4483         $timeout = 10;
4484
4485         my $count = 0;
4486         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4487             $count++;
4488             sleep 1;
4489         }
4490
4491         if ($count >= $timeout) {
4492             warn "VM still running - terminating now with SIGKILL\n";
4493             kill 9, $pid;
4494             sleep 1;
4495         }
4496
4497         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4498    });
4499 }
4500
4501 sub vm_suspend {
4502     my ($vmid, $skiplock) = @_;
4503
4504     lock_config($vmid, sub {
4505
4506         my $conf = load_config($vmid);
4507
4508         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4509
4510         vm_mon_cmd($vmid, "stop");
4511     });
4512 }
4513
4514 sub vm_resume {
4515     my ($vmid, $skiplock) = @_;
4516
4517     lock_config($vmid, sub {
4518
4519         my $conf = load_config($vmid);
4520
4521         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4522
4523         vm_mon_cmd($vmid, "cont");
4524     });
4525 }
4526
4527 sub vm_sendkey {
4528     my ($vmid, $skiplock, $key) = @_;
4529
4530     lock_config($vmid, sub {
4531
4532         my $conf = load_config($vmid);
4533
4534         # there is no qmp command, so we use the human monitor command
4535         vm_human_monitor_command($vmid, "sendkey $key");
4536     });
4537 }
4538
4539 sub vm_destroy {
4540     my ($storecfg, $vmid, $skiplock) = @_;
4541
4542     lock_config($vmid, sub {
4543
4544         my $conf = load_config($vmid);
4545
4546         check_lock($conf) if !$skiplock;
4547
4548         if (!check_running($vmid)) {
4549             fairsched_rmnod($vmid); # try to destroy group
4550             destroy_vm($storecfg, $vmid);
4551         } else {
4552             die "VM $vmid is running - destroy failed\n";
4553         }
4554     });
4555 }
4556
4557 # pci helpers
4558
4559 sub file_write {
4560     my ($filename, $buf) = @_;
4561
4562     my $fh = IO::File->new($filename, "w");
4563     return undef if !$fh;
4564
4565     my $res = print $fh $buf;
4566
4567     $fh->close();
4568
4569     return $res;
4570 }
4571
4572 sub pci_device_info {
4573     my ($name) = @_;
4574
4575     my $res;
4576
4577     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4578     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4579
4580     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4581     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4582
4583     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4584     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4585
4586     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4587     return undef if !defined($product) || $product !~ s/^0x//;
4588
4589     $res = {
4590         name => $name,
4591         vendor => $vendor,
4592         product => $product,
4593         domain => $domain,
4594         bus => $bus,
4595         slot => $slot,
4596         func => $func,
4597         irq => $irq,
4598         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4599     };
4600
4601     return $res;
4602 }
4603
4604 sub pci_dev_reset {
4605     my ($dev) = @_;
4606
4607     my $name = $dev->{name};
4608
4609     my $fn = "$pcisysfs/devices/$name/reset";
4610
4611     return file_write($fn, "1");
4612 }
4613
4614 sub pci_dev_bind_to_stub {
4615     my ($dev) = @_;
4616
4617     my $name = $dev->{name};
4618
4619     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
4620     return 1 if -d $testdir;
4621
4622     my $data = "$dev->{vendor} $dev->{product}";
4623     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
4624
4625     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4626     if (!file_write($fn, $name)) {
4627         return undef if -f $fn;
4628     }
4629
4630     $fn = "$pcisysfs/drivers/pci-stub/bind";
4631     if (! -d $testdir) {
4632         return undef if !file_write($fn, $name);
4633     }
4634
4635     return -d $testdir;
4636 }
4637
4638 sub pci_dev_bind_to_vfio {
4639     my ($dev) = @_;
4640
4641     my $name = $dev->{name};
4642
4643     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4644
4645     if (!-d $vfio_basedir) {
4646         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4647     }
4648     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4649
4650     my $testdir = "$vfio_basedir/$name";
4651     return 1 if -d $testdir;
4652
4653     my $data = "$dev->{vendor} $dev->{product}";
4654     return undef if !file_write("$vfio_basedir/new_id", $data);
4655
4656     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4657     if (!file_write($fn, $name)) {
4658         return undef if -f $fn;
4659     }
4660
4661     $fn = "$vfio_basedir/bind";
4662     if (! -d $testdir) {
4663         return undef if !file_write($fn, $name);
4664     }
4665
4666     return -d $testdir;
4667 }
4668
4669 sub pci_dev_group_bind_to_vfio {
4670     my ($pciid) = @_;
4671
4672     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4673
4674     if (!-d $vfio_basedir) {
4675         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4676     }
4677     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4678
4679     # get IOMMU group devices
4680     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4681       my @devs = grep /^0000:/, readdir($D);
4682     closedir($D);
4683
4684     foreach my $pciid (@devs) {
4685         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4686
4687         # pci bridges, switches or root ports are not supported
4688         # they have a pci_bus subdirectory so skip them
4689         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4690
4691         my $info = pci_device_info($1);
4692         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4693     }
4694
4695     return 1;
4696 }
4697
4698 sub print_pci_addr {
4699     my ($id, $bridges) = @_;
4700
4701     my $res = '';
4702     my $devices = {
4703         piix3 => { bus => 0, addr => 1 },
4704         #addr2 : first videocard
4705         balloon0 => { bus => 0, addr => 3 },
4706         watchdog => { bus => 0, addr => 4 },
4707         scsihw0 => { bus => 0, addr => 5 },
4708         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4709         scsihw1 => { bus => 0, addr => 6 },
4710         ahci0 => { bus => 0, addr => 7 },
4711         qga0 => { bus => 0, addr => 8 },
4712         spice => { bus => 0, addr => 9 },
4713         virtio0 => { bus => 0, addr => 10 },
4714         virtio1 => { bus => 0, addr => 11 },
4715         virtio2 => { bus => 0, addr => 12 },
4716         virtio3 => { bus => 0, addr => 13 },
4717         virtio4 => { bus => 0, addr => 14 },
4718         virtio5 => { bus => 0, addr => 15 },
4719         hostpci0 => { bus => 0, addr => 16 },
4720         hostpci1 => { bus => 0, addr => 17 },
4721         net0 => { bus => 0, addr => 18 },
4722         net1 => { bus => 0, addr => 19 },
4723         net2 => { bus => 0, addr => 20 },
4724         net3 => { bus => 0, addr => 21 },
4725         net4 => { bus => 0, addr => 22 },
4726         net5 => { bus => 0, addr => 23 },
4727         vga1 => { bus => 0, addr => 24 },
4728         vga2 => { bus => 0, addr => 25 },
4729         vga3 => { bus => 0, addr => 26 },
4730         hostpci2 => { bus => 0, addr => 27 },
4731         hostpci3 => { bus => 0, addr => 28 },
4732         #addr29 : usb-host (pve-usb.cfg)
4733         'pci.1' => { bus => 0, addr => 30 },
4734         'pci.2' => { bus => 0, addr => 31 },
4735         'net6' => { bus => 1, addr => 1 },
4736         'net7' => { bus => 1, addr => 2 },
4737         'net8' => { bus => 1, addr => 3 },
4738         'net9' => { bus => 1, addr => 4 },
4739         'net10' => { bus => 1, addr => 5 },
4740         'net11' => { bus => 1, addr => 6 },
4741         'net12' => { bus => 1, addr => 7 },
4742         'net13' => { bus => 1, addr => 8 },
4743         'net14' => { bus => 1, addr => 9 },
4744         'net15' => { bus => 1, addr => 10 },
4745         'net16' => { bus => 1, addr => 11 },
4746         'net17' => { bus => 1, addr => 12 },
4747         'net18' => { bus => 1, addr => 13 },
4748         'net19' => { bus => 1, addr => 14 },
4749         'net20' => { bus => 1, addr => 15 },
4750         'net21' => { bus => 1, addr => 16 },
4751         'net22' => { bus => 1, addr => 17 },
4752         'net23' => { bus => 1, addr => 18 },
4753         'net24' => { bus => 1, addr => 19 },
4754         'net25' => { bus => 1, addr => 20 },
4755         'net26' => { bus => 1, addr => 21 },
4756         'net27' => { bus => 1, addr => 22 },
4757         'net28' => { bus => 1, addr => 23 },
4758         'net29' => { bus => 1, addr => 24 },
4759         'net30' => { bus => 1, addr => 25 },
4760         'net31' => { bus => 1, addr => 26 },
4761         'virtio6' => { bus => 2, addr => 1 },
4762         'virtio7' => { bus => 2, addr => 2 },
4763         'virtio8' => { bus => 2, addr => 3 },
4764         'virtio9' => { bus => 2, addr => 4 },
4765         'virtio10' => { bus => 2, addr => 5 },
4766         'virtio11' => { bus => 2, addr => 6 },
4767         'virtio12' => { bus => 2, addr => 7 },
4768         'virtio13' => { bus => 2, addr => 8 },
4769         'virtio14' => { bus => 2, addr => 9 },
4770         'virtio15' => { bus => 2, addr => 10 },
4771         'virtioscsi0' => { bus => 3, addr => 1 },
4772         'virtioscsi1' => { bus => 3, addr => 2 },
4773         'virtioscsi2' => { bus => 3, addr => 3 },
4774         'virtioscsi3' => { bus => 3, addr => 4 },
4775         'virtioscsi4' => { bus => 3, addr => 5 },
4776         'virtioscsi5' => { bus => 3, addr => 6 },
4777         'virtioscsi6' => { bus => 3, addr => 7 },
4778         'virtioscsi7' => { bus => 3, addr => 8 },
4779         'virtioscsi8' => { bus => 3, addr => 9 },
4780         'virtioscsi9' => { bus => 3, addr => 10 },
4781         'virtioscsi10' => { bus => 3, addr => 11 },
4782         'virtioscsi11' => { bus => 3, addr => 12 },
4783         'virtioscsi12' => { bus => 3, addr => 13 },
4784         'virtioscsi13' => { bus => 3, addr => 14 },
4785         'virtioscsi14' => { bus => 3, addr => 15 },
4786         'virtioscsi15' => { bus => 3, addr => 16 },
4787         'virtioscsi16' => { bus => 3, addr => 17 },
4788         'virtioscsi17' => { bus => 3, addr => 18 },
4789         'virtioscsi18' => { bus => 3, addr => 19 },
4790         'virtioscsi19' => { bus => 3, addr => 20 },
4791         'virtioscsi20' => { bus => 3, addr => 21 },
4792         'virtioscsi21' => { bus => 3, addr => 22 },
4793         'virtioscsi22' => { bus => 3, addr => 23 },
4794         'virtioscsi23' => { bus => 3, addr => 24 },
4795         'virtioscsi24' => { bus => 3, addr => 25 },
4796         'virtioscsi25' => { bus => 3, addr => 26 },
4797         'virtioscsi26' => { bus => 3, addr => 27 },
4798         'virtioscsi27' => { bus => 3, addr => 28 },
4799         'virtioscsi28' => { bus => 3, addr => 29 },
4800         'virtioscsi29' => { bus => 3, addr => 30 },
4801         'virtioscsi30' => { bus => 3, addr => 31 },
4802
4803     };
4804
4805     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4806            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4807            my $bus = $devices->{$id}->{bus};
4808            $res = ",bus=pci.$bus,addr=$addr";
4809            $bridges->{$bus} = 1 if $bridges;
4810     }
4811     return $res;
4812
4813 }
4814
4815 sub print_pcie_addr {
4816     my ($id) = @_;
4817
4818     my $res = '';
4819     my $devices = {
4820         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4821         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4822         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4823         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4824     };
4825
4826     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4827            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4828            my $bus = $devices->{$id}->{bus};
4829            $res = ",bus=$bus,addr=$addr";
4830     }
4831     return $res;
4832
4833 }
4834
4835 # vzdump restore implementaion
4836
4837 sub tar_archive_read_firstfile {
4838     my $archive = shift;
4839
4840     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4841
4842     # try to detect archive type first
4843     my $pid = open (TMP, "tar tf '$archive'|") ||
4844         die "unable to open file '$archive'\n";
4845     my $firstfile = <TMP>;
4846     kill 15, $pid;
4847     close TMP;
4848
4849     die "ERROR: archive contaions no data\n" if !$firstfile;
4850     chomp $firstfile;
4851
4852     return $firstfile;
4853 }
4854
4855 sub tar_restore_cleanup {
4856     my ($storecfg, $statfile) = @_;
4857
4858     print STDERR "starting cleanup\n";
4859
4860     if (my $fd = IO::File->new($statfile, "r")) {
4861         while (defined(my $line = <$fd>)) {
4862             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4863                 my $volid = $2;
4864                 eval {
4865                     if ($volid =~ m|^/|) {
4866                         unlink $volid || die 'unlink failed\n';
4867                     } else {
4868                         PVE::Storage::vdisk_free($storecfg, $volid);
4869                     }
4870                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4871                 };
4872                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4873             } else {
4874                 print STDERR "unable to parse line in statfile - $line";
4875             }
4876         }
4877         $fd->close();
4878     }
4879 }
4880
4881 sub restore_archive {
4882     my ($archive, $vmid, $user, $opts) = @_;
4883
4884     my $format = $opts->{format};
4885     my $comp;
4886
4887     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4888         $format = 'tar' if !$format;
4889         $comp = 'gzip';
4890     } elsif ($archive =~ m/\.tar$/) {
4891         $format = 'tar' if !$format;
4892     } elsif ($archive =~ m/.tar.lzo$/) {
4893         $format = 'tar' if !$format;
4894         $comp = 'lzop';
4895     } elsif ($archive =~ m/\.vma$/) {
4896         $format = 'vma' if !$format;
4897     } elsif ($archive =~ m/\.vma\.gz$/) {
4898         $format = 'vma' if !$format;
4899         $comp = 'gzip';
4900     } elsif ($archive =~ m/\.vma\.lzo$/) {
4901         $format = 'vma' if !$format;
4902         $comp = 'lzop';
4903     } else {
4904         $format = 'vma' if !$format; # default
4905     }
4906
4907     # try to detect archive format
4908     if ($format eq 'tar') {
4909         return restore_tar_archive($archive, $vmid, $user, $opts);
4910     } else {
4911         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4912     }
4913 }
4914
4915 sub restore_update_config_line {
4916     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4917
4918     return if $line =~ m/^\#qmdump\#/;
4919     return if $line =~ m/^\#vzdump\#/;
4920     return if $line =~ m/^lock:/;
4921     return if $line =~ m/^unused\d+:/;
4922     return if $line =~ m/^parent:/;
4923     return if $line =~ m/^template:/; # restored VM is never a template
4924
4925     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4926         # try to convert old 1.X settings
4927         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4928         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4929             my ($model, $macaddr) = split(/\=/, $devconfig);
4930             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4931             my $net = {
4932                 model => $model,
4933                 bridge => "vmbr$ind",
4934                 macaddr => $macaddr,
4935             };
4936             my $netstr = print_net($net);
4937
4938             print $outfd "net$cookie->{netcount}: $netstr\n";
4939             $cookie->{netcount}++;
4940         }
4941     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4942         my ($id, $netstr) = ($1, $2);
4943         my $net = parse_net($netstr);
4944         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4945         $netstr = print_net($net);
4946         print $outfd "$id: $netstr\n";
4947     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4948         my $virtdev = $1;
4949         my $value = $3;
4950         if ($line =~ m/backup=no/) {
4951             print $outfd "#$line";
4952         } elsif ($virtdev && $map->{$virtdev}) {
4953             my $di = parse_drive($virtdev, $value);
4954             delete $di->{format}; # format can change on restore
4955             $di->{file} = $map->{$virtdev};
4956             $value = print_drive($vmid, $di);
4957             print $outfd "$virtdev: $value\n";
4958         } else {
4959             print $outfd $line;
4960         }
4961     } else {
4962         print $outfd $line;
4963     }
4964 }
4965
4966 sub scan_volids {
4967     my ($cfg, $vmid) = @_;
4968
4969     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4970
4971     my $volid_hash = {};
4972     foreach my $storeid (keys %$info) {
4973         foreach my $item (@{$info->{$storeid}}) {
4974             next if !($item->{volid} && $item->{size});
4975             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4976             $volid_hash->{$item->{volid}} = $item;
4977         }
4978     }
4979
4980     return $volid_hash;
4981 }
4982
4983 sub get_used_paths {
4984     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
4985
4986     my $used_path = {};
4987
4988     my $scan_config = sub {
4989         my ($cref, $snapname) = @_;
4990
4991         foreach my $key (keys %$cref) {
4992             my $value = $cref->{$key};
4993             if (valid_drivename($key)) {
4994                 next if $skip_drive && $key eq $skip_drive;
4995                 my $drive = parse_drive($key, $value);
4996                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
4997                 if ($drive->{file} =~ m!^/!) {
4998                     $used_path->{$drive->{file}}++; # = 1;
4999                 } else {
5000                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5001                     next if !$storeid;
5002                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5003                     next if !$scfg;
5004                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5005                     $used_path->{$path}++; # = 1;
5006                 }
5007             }
5008         }
5009     };
5010
5011     &$scan_config($conf);
5012
5013     undef $skip_drive;
5014
5015     if ($scan_snapshots) {
5016         foreach my $snapname (keys %{$conf->{snapshots}}) {
5017             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5018         }
5019     }
5020
5021     return $used_path;
5022 }
5023
5024 sub update_disksize {
5025     my ($vmid, $conf, $volid_hash) = @_;
5026
5027     my $changes;
5028
5029     my $used = {};
5030
5031     # Note: it is allowed to define multiple storages with same path (alias), so
5032     # we need to check both 'volid' and real 'path' (two different volid can point
5033     # to the same path).
5034
5035     my $usedpath = {};
5036
5037     # update size info
5038     foreach my $opt (keys %$conf) {
5039         if (valid_drivename($opt)) {
5040             my $drive = parse_drive($opt, $conf->{$opt});
5041             my $volid = $drive->{file};
5042             next if !$volid;
5043
5044             $used->{$volid} = 1;
5045             if ($volid_hash->{$volid} &&
5046                 (my $path = $volid_hash->{$volid}->{path})) {
5047                 $usedpath->{$path} = 1;
5048             }
5049
5050             next if drive_is_cdrom($drive);
5051             next if !$volid_hash->{$volid};
5052
5053             $drive->{size} = $volid_hash->{$volid}->{size};
5054             my $new = print_drive($vmid, $drive);
5055             if ($new ne $conf->{$opt}) {
5056                 $changes = 1;
5057                 $conf->{$opt} = $new;
5058             }
5059         }
5060     }
5061
5062     # remove 'unusedX' entry if volume is used
5063     foreach my $opt (keys %$conf) {
5064         next if $opt !~ m/^unused\d+$/;
5065         my $volid = $conf->{$opt};
5066         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5067         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5068             $changes = 1;
5069             delete $conf->{$opt};
5070         }
5071     }
5072
5073     foreach my $volid (sort keys %$volid_hash) {
5074         next if $volid =~ m/vm-$vmid-state-/;
5075         next if $used->{$volid};
5076         my $path = $volid_hash->{$volid}->{path};
5077         next if !$path; # just to be sure
5078         next if $usedpath->{$path};
5079         $changes = 1;
5080         add_unused_volume($conf, $volid);
5081         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5082     }
5083
5084     return $changes;
5085 }
5086
5087 sub rescan {
5088     my ($vmid, $nolock) = @_;
5089
5090     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5091
5092     my $volid_hash = scan_volids($cfg, $vmid);
5093
5094     my $updatefn =  sub {
5095         my ($vmid) = @_;
5096
5097         my $conf = load_config($vmid);
5098
5099         check_lock($conf);
5100
5101         my $vm_volids = {};
5102         foreach my $volid (keys %$volid_hash) {
5103             my $info = $volid_hash->{$volid};
5104             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5105         }
5106
5107         my $changes = update_disksize($vmid, $conf, $vm_volids);
5108
5109         update_config_nolock($vmid, $conf, 1) if $changes;
5110     };
5111
5112     if (defined($vmid)) {
5113         if ($nolock) {
5114             &$updatefn($vmid);
5115         } else {
5116             lock_config($vmid, $updatefn, $vmid);
5117         }
5118     } else {
5119         my $vmlist = config_list();
5120         foreach my $vmid (keys %$vmlist) {
5121             if ($nolock) {
5122                 &$updatefn($vmid);
5123             } else {
5124                 lock_config($vmid, $updatefn, $vmid);
5125             }
5126         }
5127     }
5128 }
5129
5130 sub restore_vma_archive {
5131     my ($archive, $vmid, $user, $opts, $comp) = @_;
5132
5133     my $input = $archive eq '-' ? "<&STDIN" : undef;
5134     my $readfrom = $archive;
5135
5136     my $uncomp = '';
5137     if ($comp) {
5138         $readfrom = '-';
5139         my $qarchive = PVE::Tools::shellquote($archive);
5140         if ($comp eq 'gzip') {
5141             $uncomp = "zcat $qarchive|";
5142         } elsif ($comp eq 'lzop') {
5143             $uncomp = "lzop -d -c $qarchive|";
5144         } else {
5145             die "unknown compression method '$comp'\n";
5146         }
5147
5148     }
5149
5150     my $tmpdir = "/var/tmp/vzdumptmp$$";
5151     rmtree $tmpdir;
5152
5153     # disable interrupts (always do cleanups)
5154     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5155         warn "got interrupt - ignored\n";
5156     };
5157
5158     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5159     POSIX::mkfifo($mapfifo, 0600);
5160     my $fifofh;
5161
5162     my $openfifo = sub {
5163         open($fifofh, '>', $mapfifo) || die $!;
5164     };
5165
5166     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5167
5168     my $oldtimeout;
5169     my $timeout = 5;
5170
5171     my $devinfo = {};
5172
5173     my $rpcenv = PVE::RPCEnvironment::get();
5174
5175     my $conffile = config_file($vmid);
5176     my $tmpfn = "$conffile.$$.tmp";
5177
5178     # Note: $oldconf is undef if VM does not exists
5179     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5180
5181     my $print_devmap = sub {
5182         my $virtdev_hash = {};
5183
5184         my $cfgfn = "$tmpdir/qemu-server.conf";
5185
5186         # we can read the config - that is already extracted
5187         my $fh = IO::File->new($cfgfn, "r") ||
5188             "unable to read qemu-server.conf - $!\n";
5189
5190         while (defined(my $line = <$fh>)) {
5191             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5192                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5193                 die "archive does not contain data for drive '$virtdev'\n"
5194                     if !$devinfo->{$devname};
5195                 if (defined($opts->{storage})) {
5196                     $storeid = $opts->{storage} || 'local';
5197                 } elsif (!$storeid) {
5198                     $storeid = 'local';
5199                 }
5200                 $format = 'raw' if !$format;
5201                 $devinfo->{$devname}->{devname} = $devname;
5202                 $devinfo->{$devname}->{virtdev} = $virtdev;
5203                 $devinfo->{$devname}->{format} = $format;
5204                 $devinfo->{$devname}->{storeid} = $storeid;
5205
5206                 # check permission on storage
5207                 my $pool = $opts->{pool}; # todo: do we need that?
5208                 if ($user ne 'root@pam') {
5209                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5210                 }
5211
5212                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5213             }
5214         }
5215
5216         foreach my $devname (keys %$devinfo) {
5217             die "found no device mapping information for device '$devname'\n"
5218                 if !$devinfo->{$devname}->{virtdev};
5219         }
5220
5221         my $cfg = cfs_read_file('storage.cfg');
5222
5223         # create empty/temp config
5224         if ($oldconf) {
5225             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5226             foreach_drive($oldconf, sub {
5227                 my ($ds, $drive) = @_;
5228
5229                 return if drive_is_cdrom($drive);
5230
5231                 my $volid = $drive->{file};
5232
5233                 return if !$volid || $volid =~ m|^/|;
5234
5235                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5236                 return if !$path || !$owner || ($owner != $vmid);
5237
5238                 # Note: only delete disk we want to restore
5239                 # other volumes will become unused
5240                 if ($virtdev_hash->{$ds}) {
5241                     PVE::Storage::vdisk_free($cfg, $volid);
5242                 }
5243             });
5244         }
5245
5246         my $map = {};
5247         foreach my $virtdev (sort keys %$virtdev_hash) {
5248             my $d = $virtdev_hash->{$virtdev};
5249             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5250             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5251
5252             # test if requested format is supported
5253             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5254             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5255             $d->{format} = $defFormat if !$supported;
5256
5257             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5258                                                   $d->{format}, undef, $alloc_size);
5259             print STDERR "new volume ID is '$volid'\n";
5260             $d->{volid} = $volid;
5261             my $path = PVE::Storage::path($cfg, $volid);
5262
5263             my $write_zeros = 1;
5264             # fixme: what other storages types initialize volumes with zero?
5265             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5266                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5267                 $write_zeros = 0;
5268             }
5269
5270             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5271
5272             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5273             $map->{$virtdev} = $volid;
5274         }
5275
5276         $fh->seek(0, 0) || die "seek failed - $!\n";
5277
5278         my $outfd = new IO::File ($tmpfn, "w") ||
5279             die "unable to write config for VM $vmid\n";
5280
5281         my $cookie = { netcount => 0 };
5282         while (defined(my $line = <$fh>)) {
5283             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5284         }
5285
5286         $fh->close();
5287         $outfd->close();
5288     };
5289
5290     eval {
5291         # enable interrupts
5292         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5293             die "interrupted by signal\n";
5294         };
5295         local $SIG{ALRM} = sub { die "got timeout\n"; };
5296
5297         $oldtimeout = alarm($timeout);
5298
5299         my $parser = sub {
5300             my $line = shift;
5301
5302             print "$line\n";
5303
5304             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5305                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5306                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5307             } elsif ($line =~ m/^CTIME: /) {
5308                 # we correctly received the vma config, so we can disable
5309                 # the timeout now for disk allocation (set to 10 minutes, so
5310                 # that we always timeout if something goes wrong)
5311                 alarm(600);
5312                 &$print_devmap();
5313                 print $fifofh "done\n";
5314                 my $tmp = $oldtimeout || 0;
5315                 $oldtimeout = undef;
5316                 alarm($tmp);
5317                 close($fifofh);
5318             }
5319         };
5320
5321         print "restore vma archive: $cmd\n";
5322         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5323     };
5324     my $err = $@;
5325
5326     alarm($oldtimeout) if $oldtimeout;
5327
5328     unlink $mapfifo;
5329
5330     if ($err) {
5331         rmtree $tmpdir;
5332         unlink $tmpfn;
5333
5334         my $cfg = cfs_read_file('storage.cfg');
5335         foreach my $devname (keys %$devinfo) {
5336             my $volid = $devinfo->{$devname}->{volid};
5337             next if !$volid;
5338             eval {
5339                 if ($volid =~ m|^/|) {
5340                     unlink $volid || die 'unlink failed\n';
5341                 } else {
5342                     PVE::Storage::vdisk_free($cfg, $volid);
5343                 }
5344                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5345             };
5346             print STDERR "unable to cleanup '$volid' - $@" if $@;
5347         }
5348         die $err;
5349     }
5350
5351     rmtree $tmpdir;
5352
5353     rename($tmpfn, $conffile) ||
5354         die "unable to commit configuration file '$conffile'\n";
5355
5356     PVE::Cluster::cfs_update(); # make sure we read new file
5357
5358     eval { rescan($vmid, 1); };
5359     warn $@ if $@;
5360 }
5361
5362 sub restore_tar_archive {
5363     my ($archive, $vmid, $user, $opts) = @_;
5364
5365     if ($archive ne '-') {
5366         my $firstfile = tar_archive_read_firstfile($archive);
5367         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5368             if $firstfile ne 'qemu-server.conf';
5369     }
5370
5371     my $storecfg = cfs_read_file('storage.cfg');
5372
5373     # destroy existing data - keep empty config
5374     my $vmcfgfn = config_file($vmid);
5375     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5376
5377     my $tocmd = "/usr/lib/qemu-server/qmextract";
5378
5379     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5380     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5381     $tocmd .= ' --prealloc' if $opts->{prealloc};
5382     $tocmd .= ' --info' if $opts->{info};
5383
5384     # tar option "xf" does not autodetect compression when read from STDIN,
5385     # so we pipe to zcat
5386     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5387         PVE::Tools::shellquote("--to-command=$tocmd");
5388
5389     my $tmpdir = "/var/tmp/vzdumptmp$$";
5390     mkpath $tmpdir;
5391
5392     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5393     local $ENV{VZDUMP_VMID} = $vmid;
5394     local $ENV{VZDUMP_USER} = $user;
5395
5396     my $conffile = config_file($vmid);
5397     my $tmpfn = "$conffile.$$.tmp";
5398
5399     # disable interrupts (always do cleanups)
5400     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5401         print STDERR "got interrupt - ignored\n";
5402     };
5403
5404     eval {
5405         # enable interrupts
5406         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5407             die "interrupted by signal\n";
5408         };
5409
5410         if ($archive eq '-') {
5411             print "extracting archive from STDIN\n";
5412             run_command($cmd, input => "<&STDIN");
5413         } else {
5414             print "extracting archive '$archive'\n";
5415             run_command($cmd);
5416         }
5417
5418         return if $opts->{info};
5419
5420         # read new mapping
5421         my $map = {};
5422         my $statfile = "$tmpdir/qmrestore.stat";
5423         if (my $fd = IO::File->new($statfile, "r")) {
5424             while (defined (my $line = <$fd>)) {
5425                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5426                     $map->{$1} = $2 if $1;
5427                 } else {
5428                     print STDERR "unable to parse line in statfile - $line\n";
5429                 }
5430             }
5431             $fd->close();
5432         }
5433
5434         my $confsrc = "$tmpdir/qemu-server.conf";
5435
5436         my $srcfd = new IO::File($confsrc, "r") ||
5437             die "unable to open file '$confsrc'\n";
5438
5439         my $outfd = new IO::File ($tmpfn, "w") ||
5440             die "unable to write config for VM $vmid\n";
5441
5442         my $cookie = { netcount => 0 };
5443         while (defined (my $line = <$srcfd>)) {
5444             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5445         }
5446
5447         $srcfd->close();
5448         $outfd->close();
5449     };
5450     my $err = $@;
5451
5452     if ($err) {
5453
5454         unlink $tmpfn;
5455
5456         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5457
5458         die $err;
5459     }
5460
5461     rmtree $tmpdir;
5462
5463     rename $tmpfn, $conffile ||
5464         die "unable to commit configuration file '$conffile'\n";
5465
5466     PVE::Cluster::cfs_update(); # make sure we read new file
5467
5468     eval { rescan($vmid, 1); };
5469     warn $@ if $@;
5470 };
5471
5472
5473 # Internal snapshots
5474
5475 # NOTE: Snapshot create/delete involves several non-atomic
5476 # action, and can take a long time.
5477 # So we try to avoid locking the file and use 'lock' variable
5478 # inside the config file instead.
5479
5480 my $snapshot_copy_config = sub {
5481     my ($source, $dest) = @_;
5482
5483     foreach my $k (keys %$source) {
5484         next if $k eq 'snapshots';
5485         next if $k eq 'snapstate';
5486         next if $k eq 'snaptime';
5487         next if $k eq 'vmstate';
5488         next if $k eq 'lock';
5489         next if $k eq 'digest';
5490         next if $k eq 'description';
5491         next if $k =~ m/^unused\d+$/;
5492
5493         $dest->{$k} = $source->{$k};
5494     }
5495 };
5496
5497 my $snapshot_apply_config = sub {
5498     my ($conf, $snap) = @_;
5499
5500     # copy snapshot list
5501     my $newconf = {
5502         snapshots => $conf->{snapshots},
5503     };
5504
5505     # keep description and list of unused disks
5506     foreach my $k (keys %$conf) {
5507         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5508         $newconf->{$k} = $conf->{$k};
5509     }
5510
5511     &$snapshot_copy_config($snap, $newconf);
5512
5513     return $newconf;
5514 };
5515
5516 sub foreach_writable_storage {
5517     my ($conf, $func) = @_;
5518
5519     my $sidhash = {};
5520
5521     foreach my $ds (keys %$conf) {
5522         next if !valid_drivename($ds);
5523
5524         my $drive = parse_drive($ds, $conf->{$ds});
5525         next if !$drive;
5526         next if drive_is_cdrom($drive);
5527
5528         my $volid = $drive->{file};
5529
5530         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5531         $sidhash->{$sid} = $sid if $sid;
5532     }
5533
5534     foreach my $sid (sort keys %$sidhash) {
5535         &$func($sid);
5536     }
5537 }
5538
5539 my $alloc_vmstate_volid = sub {
5540     my ($storecfg, $vmid, $conf, $snapname) = @_;
5541
5542     # Note: we try to be smart when selecting a $target storage
5543
5544     my $target;
5545
5546     # search shared storage first
5547     foreach_writable_storage($conf, sub {
5548         my ($sid) = @_;
5549         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5550         return if !$scfg->{shared};
5551
5552         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5553     });
5554
5555     if (!$target) {
5556         # now search local storage
5557         foreach_writable_storage($conf, sub {
5558             my ($sid) = @_;
5559             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5560             return if $scfg->{shared};
5561
5562             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5563         });
5564     }
5565
5566     $target = 'local' if !$target;
5567
5568     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5569     # we abort live save after $conf->{memory}, so we need at max twice that space
5570     my $size = $conf->{memory}*2 + $driver_state_size;
5571
5572     my $name = "vm-$vmid-state-$snapname";
5573     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5574     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5575     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5576
5577     return $volid;
5578 };
5579
5580 my $snapshot_prepare = sub {
5581     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5582
5583     my $snap;
5584
5585     my $updatefn =  sub {
5586
5587         my $conf = load_config($vmid);
5588
5589         die "you can't take a snapshot if it's a template\n"
5590             if is_template($conf);
5591
5592         check_lock($conf);
5593
5594         $conf->{lock} = 'snapshot';
5595
5596         die "snapshot name '$snapname' already used\n"
5597             if defined($conf->{snapshots}->{$snapname});
5598
5599         my $storecfg = PVE::Storage::config();
5600         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5601
5602         $snap = $conf->{snapshots}->{$snapname} = {};
5603
5604         if ($save_vmstate && check_running($vmid)) {
5605             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5606         }
5607
5608         &$snapshot_copy_config($conf, $snap);
5609
5610         $snap->{snapstate} = "prepare";
5611         $snap->{snaptime} = time();
5612         $snap->{description} = $comment if $comment;
5613
5614         # always overwrite machine if we save vmstate. This makes sure we
5615         # can restore it later using correct machine type
5616         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5617
5618         update_config_nolock($vmid, $conf, 1);
5619     };
5620
5621     lock_config($vmid, $updatefn);
5622
5623     return $snap;
5624 };
5625
5626 my $snapshot_commit = sub {
5627     my ($vmid, $snapname) = @_;
5628
5629     my $updatefn = sub {
5630
5631         my $conf = load_config($vmid);
5632
5633         die "missing snapshot lock\n"
5634             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5635
5636         my $has_machine_config = defined($conf->{machine});
5637
5638         my $snap = $conf->{snapshots}->{$snapname};
5639
5640         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5641
5642         die "wrong snapshot state\n"
5643             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5644
5645         delete $snap->{snapstate};
5646         delete $conf->{lock};
5647
5648         my $newconf = &$snapshot_apply_config($conf, $snap);
5649
5650         delete $newconf->{machine} if !$has_machine_config;
5651
5652         $newconf->{parent} = $snapname;
5653
5654         update_config_nolock($vmid, $newconf, 1);
5655     };
5656
5657     lock_config($vmid, $updatefn);
5658 };
5659
5660 sub snapshot_rollback {
5661     my ($vmid, $snapname) = @_;
5662
5663     my $prepare = 1;
5664
5665     my $storecfg = PVE::Storage::config();
5666
5667     my $conf = load_config($vmid);
5668
5669     my $get_snapshot_config = sub {
5670
5671         die "you can't rollback if vm is a template\n" if is_template($conf);
5672
5673         my $res = $conf->{snapshots}->{$snapname};
5674
5675         die "snapshot '$snapname' does not exist\n" if !defined($res);
5676
5677         return $res;
5678     };
5679
5680     my $snap = &$get_snapshot_config();
5681
5682     foreach_drive($snap, sub {
5683         my ($ds, $drive) = @_;
5684
5685         return if drive_is_cdrom($drive);
5686
5687         my $volid = $drive->{file};
5688
5689         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5690     });
5691
5692     my $updatefn = sub {
5693
5694         $conf = load_config($vmid);
5695
5696         $snap = &$get_snapshot_config();
5697
5698         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5699             if $snap->{snapstate};
5700
5701         if ($prepare) {
5702             check_lock($conf);
5703             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5704         }
5705
5706         die "unable to rollback vm $vmid: vm is running\n"
5707             if check_running($vmid);
5708
5709         if ($prepare) {
5710             $conf->{lock} = 'rollback';
5711         } else {
5712             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5713             delete $conf->{lock};
5714         }
5715
5716         my $forcemachine;
5717
5718         if (!$prepare) {
5719             my $has_machine_config = defined($conf->{machine});
5720
5721             # copy snapshot config to current config
5722             $conf = &$snapshot_apply_config($conf, $snap);
5723             $conf->{parent} = $snapname;
5724
5725             # Note: old code did not store 'machine', so we try to be smart
5726             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5727             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5728             # we remove the 'machine' configuration if not explicitly specified
5729             # in the original config.
5730             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5731         }
5732
5733         update_config_nolock($vmid, $conf, 1);
5734
5735         if (!$prepare && $snap->{vmstate}) {
5736             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5737             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5738         }
5739     };
5740
5741     lock_config($vmid, $updatefn);
5742
5743     foreach_drive($snap, sub {
5744         my ($ds, $drive) = @_;
5745
5746         return if drive_is_cdrom($drive);
5747
5748         my $volid = $drive->{file};
5749         my $device = "drive-$ds";
5750
5751         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5752     });
5753
5754     $prepare = 0;
5755     lock_config($vmid, $updatefn);
5756 }
5757
5758 my $savevm_wait = sub {
5759     my ($vmid) = @_;
5760
5761     for(;;) {
5762         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5763         if (!$stat->{status}) {
5764             die "savevm not active\n";
5765         } elsif ($stat->{status} eq 'active') {
5766             sleep(1);
5767             next;
5768         } elsif ($stat->{status} eq 'completed') {
5769             last;
5770         } else {
5771             die "query-savevm returned status '$stat->{status}'\n";
5772         }
5773     }
5774 };
5775
5776 sub do_snapshots_with_qemu {
5777     my ($storecfg, $volid) = @_;
5778
5779     my $storage_name = PVE::Storage::parse_volume_id($volid);
5780
5781     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}} ){
5782         return 1;
5783     }
5784
5785     if ($volid =~ m/\.(qcow2|qed)$/){
5786         return 1;
5787     }
5788
5789     return undef;
5790 }
5791
5792 sub snapshot_create {
5793     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5794
5795     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5796
5797     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5798
5799     my $config = load_config($vmid);
5800
5801     my $running = check_running($vmid);
5802
5803     my $freezefs = $running && $config->{agent};
5804     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5805
5806     my $drivehash = {};
5807
5808     if ($freezefs) {
5809         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5810         warn "guest-fsfreeze-freeze problems - $@" if $@;
5811     }
5812
5813     eval {
5814         # create internal snapshots of all drives
5815
5816         my $storecfg = PVE::Storage::config();
5817
5818         if ($running) {
5819             if ($snap->{vmstate}) {
5820                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5821                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5822                 &$savevm_wait($vmid);
5823             } else {
5824                 vm_mon_cmd($vmid, "savevm-start");
5825             }
5826         };
5827
5828         foreach_drive($snap, sub {
5829             my ($ds, $drive) = @_;
5830
5831             return if drive_is_cdrom($drive);
5832
5833             my $volid = $drive->{file};
5834             my $device = "drive-$ds";
5835
5836             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5837             $drivehash->{$ds} = 1;
5838        });
5839     };
5840     my $err = $@;
5841
5842     if ($running) {
5843         eval { vm_mon_cmd($vmid, "savevm-end")  };
5844         warn $@ if $@;
5845
5846         if ($freezefs) {
5847             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5848             warn "guest-fsfreeze-thaw problems - $@" if $@;
5849         }
5850
5851         # savevm-end is async, we need to wait
5852         for (;;) {
5853             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5854             if (!$stat->{bytes}) {
5855                 last;
5856             } else {
5857                 print "savevm not yet finished\n";
5858                 sleep(1);
5859                 next;
5860             }
5861         }
5862     }
5863
5864     if ($err) {
5865         warn "snapshot create failed: starting cleanup\n";
5866         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5867         warn $@ if $@;
5868         die $err;
5869     }
5870
5871     &$snapshot_commit($vmid, $snapname);
5872 }
5873
5874 # Note: $drivehash is only set when called from snapshot_create.
5875 sub snapshot_delete {
5876     my ($vmid, $snapname, $force, $drivehash) = @_;
5877
5878     my $prepare = 1;
5879
5880     my $snap;
5881     my $unused = [];
5882
5883     my $unlink_parent = sub {
5884         my ($confref, $new_parent) = @_;
5885
5886         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5887             if ($new_parent) {
5888                 $confref->{parent} = $new_parent;
5889             } else {
5890                 delete $confref->{parent};
5891             }
5892         }
5893     };
5894
5895     my $updatefn =  sub {
5896         my ($remove_drive) = @_;
5897
5898         my $conf = load_config($vmid);
5899
5900         if (!$drivehash) {
5901             check_lock($conf);
5902             die "you can't delete a snapshot if vm is a template\n"
5903                 if is_template($conf);
5904         }
5905
5906         $snap = $conf->{snapshots}->{$snapname};
5907
5908         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5909
5910         # remove parent refs
5911         if (!$prepare) {
5912             &$unlink_parent($conf, $snap->{parent});
5913             foreach my $sn (keys %{$conf->{snapshots}}) {
5914                 next if $sn eq $snapname;
5915                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
5916             }
5917         }
5918
5919         if ($remove_drive) {
5920             if ($remove_drive eq 'vmstate') {
5921                 delete $snap->{$remove_drive};
5922             } else {
5923                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
5924                 my $volid = $drive->{file};
5925                 delete $snap->{$remove_drive};
5926                 add_unused_volume($conf, $volid);
5927             }
5928         }
5929
5930         if ($prepare) {
5931             $snap->{snapstate} = 'delete';
5932         } else {
5933             delete $conf->{snapshots}->{$snapname};
5934             delete $conf->{lock} if $drivehash;
5935             foreach my $volid (@$unused) {
5936                 add_unused_volume($conf, $volid);
5937             }
5938         }
5939
5940         update_config_nolock($vmid, $conf, 1);
5941     };
5942
5943     lock_config($vmid, $updatefn);
5944
5945     # now remove vmstate file
5946
5947     my $storecfg = PVE::Storage::config();
5948
5949     if ($snap->{vmstate}) {
5950         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5951         if (my $err = $@) {
5952             die $err if !$force;
5953             warn $err;
5954         }
5955         # save changes (remove vmstate from snapshot)
5956         lock_config($vmid, $updatefn, 'vmstate') if !$force;
5957     };
5958
5959     # now remove all internal snapshots
5960     foreach_drive($snap, sub {
5961         my ($ds, $drive) = @_;
5962
5963         return if drive_is_cdrom($drive);
5964
5965         my $volid = $drive->{file};
5966         my $device = "drive-$ds";
5967
5968         if (!$drivehash || $drivehash->{$ds}) {
5969             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
5970             if (my $err = $@) {
5971                 die $err if !$force;
5972                 warn $err;
5973             }
5974         }
5975
5976         # save changes (remove drive fron snapshot)
5977         lock_config($vmid, $updatefn, $ds) if !$force;
5978         push @$unused, $volid;
5979     });
5980
5981     # now cleanup config
5982     $prepare = 0;
5983     lock_config($vmid, $updatefn);
5984 }
5985
5986 sub has_feature {
5987     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
5988
5989     my $err;
5990     foreach_drive($conf, sub {
5991         my ($ds, $drive) = @_;
5992
5993         return if drive_is_cdrom($drive);
5994         my $volid = $drive->{file};
5995         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
5996     });
5997
5998     return $err ? 0 : 1;
5999 }
6000
6001 sub template_create {
6002     my ($vmid, $conf, $disk) = @_;
6003
6004     my $storecfg = PVE::Storage::config();
6005
6006     foreach_drive($conf, sub {
6007         my ($ds, $drive) = @_;
6008
6009         return if drive_is_cdrom($drive);
6010         return if $disk && $ds ne $disk;
6011
6012         my $volid = $drive->{file};
6013         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6014
6015         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6016         $drive->{file} = $voliddst;
6017         $conf->{$ds} = print_drive($vmid, $drive);
6018         update_config_nolock($vmid, $conf, 1);
6019     });
6020 }
6021
6022 sub is_template {
6023     my ($conf) = @_;
6024
6025     return 1 if defined $conf->{template} && $conf->{template} == 1;
6026 }
6027
6028 sub qemu_img_convert {
6029     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6030
6031     my $storecfg = PVE::Storage::config();
6032     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6033     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6034
6035     if ($src_storeid && $dst_storeid) {
6036         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6037         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6038
6039         my $src_format = qemu_img_format($src_scfg, $src_volname);
6040         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6041
6042         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6043         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6044
6045         my $cmd = [];
6046         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6047         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6048         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6049
6050         my $parser = sub {
6051             my $line = shift;
6052             if($line =~ m/\((\S+)\/100\%\)/){
6053                 my $percent = $1;
6054                 my $transferred = int($size * $percent / 100);
6055                 my $remaining = $size - $transferred;
6056
6057                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6058             }
6059
6060         };
6061
6062         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6063         my $err = $@;
6064         die "copy failed: $err" if $err;
6065     }
6066 }
6067
6068 sub qemu_img_format {
6069     my ($scfg, $volname) = @_;
6070
6071     if ($scfg->{path} && $volname =~ m/\.(raw|qcow2|qed|vmdk)$/) {
6072         return $1;
6073     } elsif ($scfg->{type} eq 'iscsi') {
6074         return "host_device";
6075     } else {
6076         return "raw";
6077     }
6078 }
6079
6080 sub qemu_drive_mirror {
6081     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6082
6083     my $count = 0;
6084     my $old_len = 0;
6085     my $frozen = undef;
6086     my $maxwait = 120;
6087
6088     my $storecfg = PVE::Storage::config();
6089     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6090
6091     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6092
6093     my $format;
6094     if ($dst_volname =~ m/\.(raw|qcow2)$/){
6095         $format = $1;
6096     }
6097
6098     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6099
6100     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6101     $opts->{format} = $format if $format;
6102
6103     #fixme : sometime drive-mirror timeout, but works fine after.
6104     # (I have see the problem with big volume > 200GB), so we need to eval
6105     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6106     # ignore errors here
6107
6108     eval {
6109         while (1) {
6110             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6111             my $stat = @$stats[0];
6112             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6113             die "error job is not mirroring" if $stat->{type} ne "mirror";
6114
6115             my $busy = $stat->{busy};
6116
6117             if (my $total = $stat->{len}) {
6118                 my $transferred = $stat->{offset} || 0;
6119                 my $remaining = $total - $transferred;
6120                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6121
6122                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy\n";
6123             }
6124
6125             if ($stat->{len} == $stat->{offset}) {
6126                 if ($busy eq 'false') {
6127
6128                     last if $vmiddst != $vmid;
6129
6130                     # try to switch the disk if source and destination are on the same guest
6131                     eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6132                     last if !$@;
6133                     die $@ if $@ !~ m/cannot be completed/;
6134                 }
6135
6136                 if ($count > $maxwait) {
6137                     # if too much writes to disk occurs at the end of migration
6138                     #the disk needs to be freezed to be able to complete the migration
6139                     vm_suspend($vmid,1);
6140                     $frozen = 1;
6141                 }
6142                 $count ++
6143             }
6144             $old_len = $stat->{offset};
6145             sleep 1;
6146         }
6147
6148         vm_resume($vmid, 1) if $frozen;
6149
6150     };
6151     my $err = $@;
6152
6153     my $cancel_job = sub {
6154         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6155         while (1) {
6156             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6157             my $stat = @$stats[0];
6158             last if !$stat;
6159             sleep 1;
6160         }
6161     };
6162
6163     if ($err) {
6164         eval { &$cancel_job(); };
6165         die "mirroring error: $err";
6166     }
6167
6168     if ($vmiddst != $vmid) {
6169         # if we clone a disk for a new target vm, we don't switch the disk
6170         &$cancel_job(); # so we call block-job-cancel
6171     }
6172 }
6173
6174 sub clone_disk {
6175     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6176         $newvmid, $storage, $format, $full, $newvollist) = @_;
6177
6178     my $newvolid;
6179
6180     if (!$full) {
6181         print "create linked clone of drive $drivename ($drive->{file})\n";
6182         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6183         push @$newvollist, $newvolid;
6184     } else {
6185         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6186         $storeid = $storage if $storage;
6187
6188         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6189         if (!$format) {
6190             $format = $drive->{format} || $defFormat;
6191         }
6192
6193         # test if requested format is supported - else use default
6194         my $supported = grep { $_ eq $format } @$validFormats;
6195         $format = $defFormat if !$supported;
6196
6197         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6198
6199         print "create full clone of drive $drivename ($drive->{file})\n";
6200         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6201         push @$newvollist, $newvolid;
6202
6203         if (!$running || $snapname) {
6204             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6205         } else {
6206             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6207         }
6208     }
6209
6210     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6211
6212     my $disk = $drive;
6213     $disk->{format} = undef;
6214     $disk->{file} = $newvolid;
6215     $disk->{size} = $size;
6216
6217     return $disk;
6218 }
6219
6220 # this only works if VM is running
6221 sub get_current_qemu_machine {
6222     my ($vmid) = @_;
6223
6224     my $cmd = { execute => 'query-machines', arguments => {} };
6225     my $res = vm_qmp_command($vmid, $cmd);
6226
6227     my ($current, $default);
6228     foreach my $e (@$res) {
6229         $default = $e->{name} if $e->{'is-default'};
6230         $current = $e->{name} if $e->{'is-current'};
6231     }
6232
6233     # fallback to the default machine if current is not supported by qemu
6234     return $current || $default || 'pc';
6235 }
6236
6237 sub qemu_machine_feature_enabled {
6238     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6239
6240     my $current_major;
6241     my $current_minor;
6242
6243     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6244
6245         $current_major = $3;
6246         $current_minor = $4;
6247
6248     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6249
6250         $current_major = $1;
6251         $current_minor = $2;
6252     }
6253
6254     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6255
6256
6257 }
6258
6259 sub lspci {
6260
6261     my $devices = {};
6262
6263     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6264             my (undef, $id, $function) = @_;
6265             my $res = { id => $id, function => $function};
6266             push @{$devices->{$id}}, $res;
6267     });
6268
6269     return $devices;
6270 }
6271
6272 sub vm_iothreads_list {
6273     my ($vmid) = @_;
6274
6275     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6276
6277     my $iothreads = {};
6278     foreach my $iothread (@$res) {
6279         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6280     }
6281
6282     return $iothreads;
6283 }
6284
6285 sub scsihw_infos {
6286     my ($conf, $drive) = @_;
6287
6288     my $maxdev = 0;
6289
6290     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6291         $maxdev = 7;
6292     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6293         $maxdev = 1;
6294     } else {
6295         $maxdev = 256;
6296     }
6297
6298     my $controller = int($drive->{index} / $maxdev);
6299     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6300
6301     return ($maxdev, $controller, $controller_prefix);
6302 }
6303
6304 1;