PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $root_path = "/sys/fs/cgroup/";
  74    my $vm_path = $root_path.$controller."/qemu.slice/$vmid.scope";
  75    return if !$vm_path;
  76
  77    PVE::ProcFSTools::write_proc_entry("$vm_path/$option", $value);
  78
  79 }
  80
  81 unless(defined(&_VZSYSCALLS_H_)) {
  82     eval 'sub _VZSYSCALLS_H_ () {1;}' unless defined(&_VZSYSCALLS_H_);
  83     require 'sys/syscall.ph';
  84     if(defined(&__x86_64__)) {
  85         eval 'sub __NR_fairsched_vcpus () {499;}' unless defined(&__NR_fairsched_vcpus);
  86         eval 'sub __NR_fairsched_mknod () {504;}' unless defined(&__NR_fairsched_mknod);
  87         eval 'sub __NR_fairsched_rmnod () {505;}' unless defined(&__NR_fairsched_rmnod);
  88         eval 'sub __NR_fairsched_chwt () {506;}' unless defined(&__NR_fairsched_chwt);
  89         eval 'sub __NR_fairsched_mvpr () {507;}' unless defined(&__NR_fairsched_mvpr);
  90         eval 'sub __NR_fairsched_rate () {508;}' unless defined(&__NR_fairsched_rate);
  91         eval 'sub __NR_setluid () {501;}' unless defined(&__NR_setluid);
  92         eval 'sub __NR_setublimit () {502;}' unless defined(&__NR_setublimit);
  93     }
  94     elsif(defined( &__i386__) ) {
  95         eval 'sub __NR_fairsched_mknod () {500;}' unless defined(&__NR_fairsched_mknod);
  96         eval 'sub __NR_fairsched_rmnod () {501;}' unless defined(&__NR_fairsched_rmnod);
  97         eval 'sub __NR_fairsched_chwt () {502;}' unless defined(&__NR_fairsched_chwt);
  98         eval 'sub __NR_fairsched_mvpr () {503;}' unless defined(&__NR_fairsched_mvpr);
  99         eval 'sub __NR_fairsched_rate () {504;}' unless defined(&__NR_fairsched_rate);
 100         eval 'sub __NR_fairsched_vcpus () {505;}' unless defined(&__NR_fairsched_vcpus);
 101         eval 'sub __NR_setluid () {511;}' unless defined(&__NR_setluid);
 102         eval 'sub __NR_setublimit () {512;}' unless defined(&__NR_setublimit);
 103     } else {
 104         die("no fairsched syscall for this arch");
 105     }
 106     require 'asm/ioctl.ph';
 107     eval 'sub KVM_GET_API_VERSION () { &_IO(0xAE, 0x);}' unless defined(&KVM_GET_API_VERSION);
 108 }
 109
 110 sub fairsched_mknod {
 111     my ($parent, $weight, $desired) = @_;
 112
 113     return syscall(&__NR_fairsched_mknod, int($parent), int($weight), int($desired));
 114 }
 115
 116 sub fairsched_rmnod {
 117     my ($id) = @_;
 118
 119     return syscall(&__NR_fairsched_rmnod, int($id));
 120 }
 121
 122 sub fairsched_mvpr {
 123     my ($pid, $newid) = @_;
 124
 125     return syscall(&__NR_fairsched_mvpr, int($pid), int($newid));
 126 }
 127
 128 sub fairsched_vcpus {
 129     my ($id, $vcpus) = @_;
 130
 131     return syscall(&__NR_fairsched_vcpus, int($id), int($vcpus));
 132 }
 133
 134 sub fairsched_rate {
 135     my ($id, $op, $rate) = @_;
 136
 137     return syscall(&__NR_fairsched_rate, int($id), int($op), int($rate));
 138 }
 139
 140 use constant FAIRSCHED_SET_RATE  => 0;
 141 use constant FAIRSCHED_DROP_RATE => 1;
 142 use constant FAIRSCHED_GET_RATE  => 2;
 143
 144 sub fairsched_cpulimit {
 145     my ($id, $limit) = @_;
 146
 147     my $cpulim1024 = int($limit * 1024 / 100);
 148     my $op = $cpulim1024 ? FAIRSCHED_SET_RATE : FAIRSCHED_DROP_RATE;
 149
 150     return fairsched_rate($id, $op, $cpulim1024);
 151 }
 152
 153 my $nodename = PVE::INotify::nodename();
 154
 155 mkdir "/etc/pve/nodes/$nodename";
 156 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 157 mkdir $confdir;
 158
 159 my $var_run_tmpdir = "/var/run/qemu-server";
 160 mkdir $var_run_tmpdir;
 161
 162 my $lock_dir = "/var/lock/qemu-server";
 163 mkdir $lock_dir;
 164
 165 my $pcisysfs = "/sys/bus/pci";
 166
 167 my $confdesc = {
 168     onboot => {
 169         optional => 1,
 170         type => 'boolean',
 171         description => "Specifies whether a VM will be started during system bootup.",
 172         default => 0,
 173     },
 174     autostart => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Automatic restart after crash (currently ignored).",
 178         default => 0,
 179     },
 180     hotplug => {
 181         optional => 1,
 182         type => 'string', format => 'pve-hotplug-features',
 183         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 184         default => 'network,disk,usb',
 185     },
 186     reboot => {
 187         optional => 1,
 188         type => 'boolean',
 189         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 190         default => 1,
 191     },
 192     lock => {
 193         optional => 1,
 194         type => 'string',
 195         description => "Lock/unlock the VM.",
 196         enum => [qw(migrate backup snapshot rollback)],
 197     },
 198     cpulimit => {
 199         optional => 1,
 200         type => 'integer',
 201         description => "Limit of CPU usage in per cent. Note if the computer has 2 CPUs, it has total of 200% CPU time. Value '0' indicates no CPU limit.\n\nNOTE: This option is currently ignored.",
 202         minimum => 0,
 203         default => 0,
 204     },
 205     cpuunits => {
 206         optional => 1,
 207         type => 'integer',
 208         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 209         minimum => 0,
 210         maximum => 500000,
 211         default => 1000,
 212     },
 213     memory => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 217         minimum => 16,
 218         default => 512,
 219     },
 220     balloon => {
 221         optional => 1,
 222         type => 'integer',
 223         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 224         minimum => 0,
 225     },
 226     shares => {
 227         optional => 1,
 228         type => 'integer',
 229         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 230         minimum => 0,
 231         maximum => 50000,
 232         default => 1000,
 233     },
 234     keyboard => {
 235         optional => 1,
 236         type => 'string',
 237         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 238         enum => PVE::Tools::kvmkeymaplist(),
 239         default => 'en-us',
 240     },
 241     name => {
 242         optional => 1,
 243         type => 'string', format => 'dns-name',
 244         description => "Set a name for the VM. Only used on the configuration web interface.",
 245     },
 246     scsihw => {
 247         optional => 1,
 248         type => 'string',
 249         description => "scsi controller model",
 250         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 251         default => 'lsi',
 252     },
 253     description => {
 254         optional => 1,
 255         type => 'string',
 256         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 257     },
 258     ostype => {
 259         optional => 1,
 260         type => 'string',
 261         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 262         description => <<EODESC,
 263 Used to enable special optimization/features for specific
 264 operating systems:
 265
 266 other  => unspecified OS
 267 wxp    => Microsoft Windows XP
 268 w2k    => Microsoft Windows 2000
 269 w2k3   => Microsoft Windows 2003
 270 w2k8   => Microsoft Windows 2008
 271 wvista => Microsoft Windows Vista
 272 win7   => Microsoft Windows 7
 273 win8   => Microsoft Windows 8/2012
 274 l24    => Linux 2.4 Kernel
 275 l26    => Linux 2.6/3.X Kernel
 276 solaris => solaris/opensolaris/openindiania kernel
 277
 278 other|l24|l26|solaris                       ... no special behaviour
 279 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 280 EODESC
 281     },
 282     boot => {
 283         optional => 1,
 284         type => 'string',
 285         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 286         pattern => '[acdn]{1,4}',
 287         default => 'cdn',
 288     },
 289     bootdisk => {
 290         optional => 1,
 291         type => 'string', format => 'pve-qm-bootdisk',
 292         description => "Enable booting from specified disk.",
 293         pattern => '(ide|sata|scsi|virtio)\d+',
 294     },
 295     smp => {
 296         optional => 1,
 297         type => 'integer',
 298         description => "The number of CPUs. Please use option -sockets instead.",
 299         minimum => 1,
 300         default => 1,
 301     },
 302     sockets => {
 303         optional => 1,
 304         type => 'integer',
 305         description => "The number of CPU sockets.",
 306         minimum => 1,
 307         default => 1,
 308     },
 309     cores => {
 310         optional => 1,
 311         type => 'integer',
 312         description => "The number of cores per socket.",
 313         minimum => 1,
 314         default => 1,
 315     },
 316     numa => {
 317         optional => 1,
 318         type => 'boolean',
 319         description => "Enable/disable Numa.",
 320         default => 0,
 321     },
 322     vcpus => {
 323         optional => 1,
 324         type => 'integer',
 325         description => "Number of hotplugged vcpus.",
 326         minimum => 1,
 327         default => 0,
 328     },
 329     acpi => {
 330         optional => 1,
 331         type => 'boolean',
 332         description => "Enable/disable ACPI.",
 333         default => 1,
 334     },
 335     agent => {
 336         optional => 1,
 337         type => 'boolean',
 338         description => "Enable/disable Qemu GuestAgent.",
 339         default => 0,
 340     },
 341     kvm => {
 342         optional => 1,
 343         type => 'boolean',
 344         description => "Enable/disable KVM hardware virtualization.",
 345         default => 1,
 346     },
 347     tdf => {
 348         optional => 1,
 349         type => 'boolean',
 350         description => "Enable/disable time drift fix.",
 351         default => 0,
 352     },
 353     localtime => {
 354         optional => 1,
 355         type => 'boolean',
 356         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 357     },
 358     freeze => {
 359         optional => 1,
 360         type => 'boolean',
 361         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 362     },
 363     vga => {
 364         optional => 1,
 365         type => 'string',
 366         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 367         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 368     },
 369     watchdog => {
 370         optional => 1,
 371         type => 'string', format => 'pve-qm-watchdog',
 372         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 373         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 374     },
 375     startdate => {
 376         optional => 1,
 377         type => 'string',
 378         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 379         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 380         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 381         default => 'now',
 382     },
 383     startup =>  get_standard_option('pve-startup-order'),
 384     template => {
 385         optional => 1,
 386         type => 'boolean',
 387         description => "Enable/disable Template.",
 388         default => 0,
 389     },
 390     args => {
 391         optional => 1,
 392         type => 'string',
 393         description => <<EODESCR,
 394 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 395
 396 args: -no-reboot -no-hpet
 397 EODESCR
 398     },
 399     tablet => {
 400         optional => 1,
 401         type => 'boolean',
 402         default => 1,
 403         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 404     },
 405     migrate_speed => {
 406         optional => 1,
 407         type => 'integer',
 408         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 409         minimum => 0,
 410         default => 0,
 411     },
 412     migrate_downtime => {
 413         optional => 1,
 414         type => 'number',
 415         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 416         minimum => 0,
 417         default => 0.1,
 418     },
 419     cdrom => {
 420         optional => 1,
 421         type => 'string', format => 'pve-qm-drive',
 422         typetext => 'volume',
 423         description => "This is an alias for option -ide2",
 424     },
 425     cpu => {
 426         optional => 1,
 427         description => "Emulated CPU type.",
 428         type => 'string',
 429         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 430         default => 'kvm64',
 431     },
 432     parent => get_standard_option('pve-snapshot-name', {
 433         optional => 1,
 434         description => "Parent snapshot name. This is used internally, and should not be modified.",
 435     }),
 436     snaptime => {
 437         optional => 1,
 438         description => "Timestamp for snapshots.",
 439         type => 'integer',
 440         minimum => 0,
 441     },
 442     vmstate => {
 443         optional => 1,
 444         type => 'string', format => 'pve-volume-id',
 445         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 446     },
 447     machine => {
 448         description => "Specific the Qemu machine type.",
 449         type => 'string',
 450         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 451         maxLength => 40,
 452         optional => 1,
 453     },
 454     smbios1 => {
 455         description => "Specify SMBIOS type 1 fields.",
 456         type => 'string', format => 'pve-qm-smbios1',
 457         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 458         maxLength => 256,
 459         optional => 1,
 460     },
 461 };
 462
 463 # what about other qemu settings ?
 464 #cpu => 'string',
 465 #machine => 'string',
 466 #fda => 'file',
 467 #fdb => 'file',
 468 #mtdblock => 'file',
 469 #sd => 'file',
 470 #pflash => 'file',
 471 #snapshot => 'bool',
 472 #bootp => 'file',
 473 ##tftp => 'dir',
 474 ##smb => 'dir',
 475 #kernel => 'file',
 476 #append => 'string',
 477 #initrd => 'file',
 478 ##soundhw => 'string',
 479
 480 while (my ($k, $v) = each %$confdesc) {
 481     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 482 }
 483
 484 my $MAX_IDE_DISKS = 4;
 485 my $MAX_SCSI_DISKS = 14;
 486 my $MAX_VIRTIO_DISKS = 16;
 487 my $MAX_SATA_DISKS = 6;
 488 my $MAX_USB_DEVICES = 5;
 489 my $MAX_NETS = 32;
 490 my $MAX_UNUSED_DISKS = 8;
 491 my $MAX_HOSTPCI_DEVICES = 4;
 492 my $MAX_SERIAL_PORTS = 4;
 493 my $MAX_PARALLEL_PORTS = 3;
 494 my $MAX_NUMA = 8;
 495 my $MAX_MEM = 4194304;
 496 my $STATICMEM = 1024;
 497
 498 my $numadesc = {
 499     optional => 1,
 500     type => 'string', format => 'pve-qm-numanode',
 501     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 502     description => "numa topology",
 503 };
 504 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 505
 506 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 507     $confdesc->{"numa$i"} = $numadesc;
 508 }
 509
 510 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 511                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 512                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 513 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 514
 515 my $netdesc = {
 516     optional => 1,
 517     type => 'string', format => 'pve-qm-net',
 518     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 519     description => <<EODESCR,
 520 Specify network devices.
 521
 522 MODEL is one of: $nic_model_list_txt
 523
 524 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 525 automatically generated if not specified.
 526
 527 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 528
 529 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 530
 531 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 532
 533 10.0.2.2   Gateway
 534 10.0.2.3   DNS Server
 535 10.0.2.4   SMB Server
 536
 537 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 538
 539 EODESCR
 540 };
 541 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 542
 543 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 544     $confdesc->{"net$i"} = $netdesc;
 545 }
 546
 547 my $drivename_hash;
 548
 549 my $idedesc = {
 550     optional => 1,
 551     type => 'string', format => 'pve-qm-drive',
 552     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 553     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 554 };
 555 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 556
 557 my $scsidesc = {
 558     optional => 1,
 559     type => 'string', format => 'pve-qm-drive',
 560     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>]',
 561     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 562 };
 563 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 564
 565 my $satadesc = {
 566     optional => 1,
 567     type => 'string', format => 'pve-qm-drive',
 568     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 569     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 570 };
 571 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 572
 573 my $virtiodesc = {
 574     optional => 1,
 575     type => 'string', format => 'pve-qm-drive',
 576     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
 577     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 578 };
 579 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 580
 581 my $usbdesc = {
 582     optional => 1,
 583     type => 'string', format => 'pve-qm-usb-device',
 584     typetext => 'host=HOSTUSBDEVICE|spice',
 585     description => <<EODESCR,
 586 Configure an USB device (n is 0 to 4). This can be used to
 587 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 588
 589 'bus-port(.port)*' (decimal numbers) or
 590 'vendor_id:product_id' (hexadeciaml numbers)
 591
 592 You can use the 'lsusb -t' command to list existing usb devices.
 593
 594 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 595
 596 The value 'spice' can be used to add a usb redirection devices for spice.
 597
 598 EODESCR
 599 };
 600 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 601
 602 my $hostpcidesc = {
 603         optional => 1,
 604         type => 'string', format => 'pve-qm-hostpci',
 605         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 606         description => <<EODESCR,
 607 Map host pci devices. HOSTPCIDEVICE syntax is:
 608
 609 'bus:dev.func' (hexadecimal numbers)
 610
 611 You can us the 'lspci' command to list existing pci devices.
 612
 613 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 614
 615 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 616
 617 Experimental: user reported problems with this option.
 618 EODESCR
 619 };
 620 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 621
 622 my $serialdesc = {
 623         optional => 1,
 624         type => 'string',
 625         pattern => '(/dev/.+|socket)',
 626         description =>  <<EODESCR,
 627 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 628
 629 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 630
 631 Experimental: user reported problems with this option.
 632 EODESCR
 633 };
 634
 635 my $paralleldesc= {
 636         optional => 1,
 637         type => 'string',
 638         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 639         description =>  <<EODESCR,
 640 Map host parallel devices (n is 0 to 2).
 641
 642 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 643
 644 Experimental: user reported problems with this option.
 645 EODESCR
 646 };
 647
 648 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 649     $confdesc->{"parallel$i"} = $paralleldesc;
 650 }
 651
 652 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 653     $confdesc->{"serial$i"} = $serialdesc;
 654 }
 655
 656 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 657     $confdesc->{"hostpci$i"} = $hostpcidesc;
 658 }
 659
 660 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 661     $drivename_hash->{"ide$i"} = 1;
 662     $confdesc->{"ide$i"} = $idedesc;
 663 }
 664
 665 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 666     $drivename_hash->{"sata$i"} = 1;
 667     $confdesc->{"sata$i"} = $satadesc;
 668 }
 669
 670 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 671     $drivename_hash->{"scsi$i"} = 1;
 672     $confdesc->{"scsi$i"} = $scsidesc ;
 673 }
 674
 675 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 676     $drivename_hash->{"virtio$i"} = 1;
 677     $confdesc->{"virtio$i"} = $virtiodesc;
 678 }
 679
 680 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 681     $confdesc->{"usb$i"} = $usbdesc;
 682 }
 683
 684 my $unuseddesc = {
 685     optional => 1,
 686     type => 'string', format => 'pve-volume-id',
 687     description => "Reference to unused volumes.",
 688 };
 689
 690 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 691     $confdesc->{"unused$i"} = $unuseddesc;
 692 }
 693
 694 my $kvm_api_version = 0;
 695
 696 sub kvm_version {
 697
 698     return $kvm_api_version if $kvm_api_version;
 699
 700     my $fh = IO::File->new("</dev/kvm") ||
 701         return 0;
 702
 703     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 704         $kvm_api_version = $v;
 705     }
 706
 707     $fh->close();
 708
 709     return  $kvm_api_version;
 710 }
 711
 712 my $kvm_user_version;
 713
 714 sub kvm_user_version {
 715
 716     return $kvm_user_version if $kvm_user_version;
 717
 718     $kvm_user_version = 'unknown';
 719
 720     my $tmp = `kvm -help 2>/dev/null`;
 721
 722     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 723         $kvm_user_version = $2;
 724     }
 725
 726     return $kvm_user_version;
 727
 728 }
 729
 730 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 731
 732 sub disknames {
 733     # order is important - used to autoselect boot disk
 734     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 735             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 736             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 737             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 738 }
 739
 740 sub valid_drivename {
 741     my $dev = shift;
 742
 743     return defined($drivename_hash->{$dev});
 744 }
 745
 746 sub option_exists {
 747     my $key = shift;
 748     return defined($confdesc->{$key});
 749 }
 750
 751 sub nic_models {
 752     return $nic_model_list;
 753 }
 754
 755 sub os_list_description {
 756
 757     return {
 758         other => 'Other',
 759         wxp => 'Windows XP',
 760         w2k => 'Windows 2000',
 761         w2k3 =>, 'Windows 2003',
 762         w2k8 => 'Windows 2008',
 763         wvista => 'Windows Vista',
 764         win7 => 'Windows 7',
 765         win8 => 'Windows 8/2012',
 766         l24 => 'Linux 2.4',
 767         l26 => 'Linux 2.6',
 768     };
 769 }
 770
 771 my $cdrom_path;
 772
 773 sub get_cdrom_path {
 774
 775     return  $cdrom_path if $cdrom_path;
 776
 777     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 778     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 779     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 780 }
 781
 782 sub get_iso_path {
 783     my ($storecfg, $vmid, $cdrom) = @_;
 784
 785     if ($cdrom eq 'cdrom') {
 786         return get_cdrom_path();
 787     } elsif ($cdrom eq 'none') {
 788         return '';
 789     } elsif ($cdrom =~ m|^/|) {
 790         return $cdrom;
 791     } else {
 792         return PVE::Storage::path($storecfg, $cdrom);
 793     }
 794 }
 795
 796 # try to convert old style file names to volume IDs
 797 sub filename_to_volume_id {
 798     my ($vmid, $file, $media) = @_;
 799
 800     if (!($file eq 'none' || $file eq 'cdrom' ||
 801           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 802
 803         return undef if $file =~ m|/|;
 804
 805         if ($media && $media eq 'cdrom') {
 806             $file = "local:iso/$file";
 807         } else {
 808             $file = "local:$vmid/$file";
 809         }
 810     }
 811
 812     return $file;
 813 }
 814
 815 sub verify_media_type {
 816     my ($opt, $vtype, $media) = @_;
 817
 818     return if !$media;
 819
 820     my $etype;
 821     if ($media eq 'disk') {
 822         $etype = 'images';
 823     } elsif ($media eq 'cdrom') {
 824         $etype = 'iso';
 825     } else {
 826         die "internal error";
 827     }
 828
 829     return if ($vtype eq $etype);
 830
 831     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 832 }
 833
 834 sub cleanup_drive_path {
 835     my ($opt, $storecfg, $drive) = @_;
 836
 837     # try to convert filesystem paths to volume IDs
 838
 839     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 840         ($drive->{file} !~ m|^/dev/.+|) &&
 841         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 842         ($drive->{file} !~ m/^\d+$/)) {
 843         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 844         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 845         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 846         verify_media_type($opt, $vtype, $drive->{media});
 847         $drive->{file} = $volid;
 848     }
 849
 850     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 851 }
 852
 853 sub create_conf_nolock {
 854     my ($vmid, $settings) = @_;
 855
 856     my $filename = config_file($vmid);
 857
 858     die "configuration file '$filename' already exists\n" if -f $filename;
 859
 860     my $defaults = load_defaults();
 861
 862     $settings->{name} = "vm$vmid" if !$settings->{name};
 863     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 864
 865     my $data = '';
 866     foreach my $opt (keys %$settings) {
 867         next if !$confdesc->{$opt};
 868
 869         my $value = $settings->{$opt};
 870         next if !$value;
 871
 872         $data .= "$opt: $value\n";
 873     }
 874
 875     PVE::Tools::file_set_contents($filename, $data);
 876 }
 877
 878 sub parse_hotplug_features {
 879     my ($data) = @_;
 880
 881     my $res = {};
 882
 883     return $res if $data eq '0';
 884
 885     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 886
 887     foreach my $feature (PVE::Tools::split_list($data)) {
 888         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 889             $res->{$1} = 1;
 890         } else {
 891             warn "ignoring unknown hotplug feature '$feature'\n";
 892         }
 893     }
 894     return $res;
 895 }
 896
 897 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 898 sub pve_verify_hotplug_features {
 899     my ($value, $noerr) = @_;
 900
 901     return $value if parse_hotplug_features($value);
 902
 903     return undef if $noerr;
 904
 905     die "unable to parse hotplug option\n";
 906 }
 907
 908 my $parse_size = sub {
 909     my ($value) = @_;
 910
 911     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 912     my ($size, $unit) = ($1, $3);
 913     if ($unit) {
 914         if ($unit eq 'K') {
 915             $size = $size * 1024;
 916         } elsif ($unit eq 'M') {
 917             $size = $size * 1024 * 1024;
 918         } elsif ($unit eq 'G') {
 919             $size = $size * 1024 * 1024 * 1024;
 920         }
 921     }
 922     return int($size);
 923 };
 924
 925 my $format_size = sub {
 926     my ($size) = @_;
 927
 928     $size = int($size);
 929
 930     my $kb = int($size/1024);
 931     return $size if $kb*1024 != $size;
 932
 933     my $mb = int($kb/1024);
 934     return "${kb}K" if $mb*1024 != $kb;
 935
 936     my $gb = int($mb/1024);
 937     return "${mb}M" if $gb*1024 != $mb;
 938
 939     return "${gb}G";
 940 };
 941
 942 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 943 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 944 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 945 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 946
 947 sub parse_drive {
 948     my ($key, $data) = @_;
 949
 950     my $res = {};
 951
 952     # $key may be undefined - used to verify JSON parameters
 953     if (!defined($key)) {
 954         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 955         $res->{index} = 0;
 956     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 957         $res->{interface} = $1;
 958         $res->{index} = $2;
 959     } else {
 960         return undef;
 961     }
 962
 963     foreach my $p (split (/,/, $data)) {
 964         next if $p =~ m/^\s*$/;
 965
 966         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues)=(.+)$/) {
 967             my ($k, $v) = ($1, $2);
 968
 969             $k = 'file' if $k eq 'volume';
 970
 971             return undef if defined $res->{$k};
 972
 973             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 974                 return undef if !$v || $v !~ m/^\d+/;
 975                 $k = "m$k";
 976                 $v = sprintf("%.3f", $v / (1024*1024));
 977             }
 978             $res->{$k} = $v;
 979         } else {
 980             if (!$res->{file} && $p !~ m/=/) {
 981                 $res->{file} = $p;
 982             } else {
 983                 return undef;
 984             }
 985         }
 986     }
 987
 988     return undef if !$res->{file};
 989
 990     if($res->{file} =~ m/\.(raw|cow|qcow|qcow2|vmdk|cloop)$/){
 991         $res->{format} = $1;
 992     }
 993
 994     return undef if $res->{cache} &&
 995         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 996     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 997     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 998     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 999     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
1000     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
1001     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
1002     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qcow2|vmdk|cloop)$/;
1003     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
1004     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
1005     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
1006     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
1007     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
1008     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
1009     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
1010
1011     return undef if $res->{mbps_rd} && $res->{mbps};
1012     return undef if $res->{mbps_wr} && $res->{mbps};
1013
1014     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
1015     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
1016     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
1017     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
1018     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
1019     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
1020
1021     return undef if $res->{iops_rd} && $res->{iops};
1022     return undef if $res->{iops_wr} && $res->{iops};
1023
1024
1025     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
1026     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
1027     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
1028     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
1029     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
1030     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
1031
1032
1033     if ($res->{size}) {
1034         return undef if !defined($res->{size} = &$parse_size($res->{size}));
1035     }
1036
1037     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1038         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1039         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1040         return undef if $res->{interface} eq 'virtio';
1041     }
1042
1043     # rerror does not work with scsi drives
1044     if ($res->{rerror}) {
1045         return undef if $res->{interface} eq 'scsi';
1046     }
1047
1048     return $res;
1049 }
1050
1051 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1052
1053 sub print_drive {
1054     my ($vmid, $drive) = @_;
1055
1056     my $opts = '';
1057     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
1058         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1059     }
1060
1061     if ($drive->{size}) {
1062         $opts .= ",size=" . &$format_size($drive->{size});
1063     }
1064
1065     return "$drive->{file}$opts";
1066 }
1067
1068 sub scsi_inquiry {
1069     my($fh, $noerr) = @_;
1070
1071     my $SG_IO = 0x2285;
1072     my $SG_GET_VERSION_NUM = 0x2282;
1073
1074     my $versionbuf = "\x00" x 8;
1075     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1076     if (!$ret) {
1077         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1078         return undef;
1079     }
1080     my $version = unpack("I", $versionbuf);
1081     if ($version < 30000) {
1082         die "scsi generic interface too old\n"  if !$noerr;
1083         return undef;
1084     }
1085
1086     my $buf = "\x00" x 36;
1087     my $sensebuf = "\x00" x 8;
1088     my $cmd = pack("C x3 C x1", 0x12, 36);
1089
1090     # see /usr/include/scsi/sg.h
1091     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1092
1093     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1094                       length($sensebuf), 0, length($buf), $buf,
1095                       $cmd, $sensebuf, 6000);
1096
1097     $ret = ioctl($fh, $SG_IO, $packet);
1098     if (!$ret) {
1099         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1100         return undef;
1101     }
1102
1103     my @res = unpack($sg_io_hdr_t, $packet);
1104     if ($res[17] || $res[18]) {
1105         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1106         return undef;
1107     }
1108
1109     my $res = {};
1110     (my $byte0, my $byte1, $res->{vendor},
1111      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1112
1113     $res->{removable} = $byte1 & 128 ? 1 : 0;
1114     $res->{type} = $byte0 & 31;
1115
1116     return $res;
1117 }
1118
1119 sub path_is_scsi {
1120     my ($path) = @_;
1121
1122     my $fh = IO::File->new("+<$path") || return undef;
1123     my $res = scsi_inquiry($fh, 1);
1124     close($fh);
1125
1126     return $res;
1127 }
1128
1129 sub machine_type_is_q35 {
1130     my ($conf) = @_;
1131
1132     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1133 }
1134
1135 sub print_tabletdevice_full {
1136     my ($conf) = @_;
1137
1138     my $q35 = machine_type_is_q35($conf);
1139
1140     # we use uhci for old VMs because tablet driver was buggy in older qemu
1141     my $usbbus = $q35 ? "ehci" : "uhci";
1142
1143     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1144 }
1145
1146 sub print_drivedevice_full {
1147     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1148
1149     my $device = '';
1150     my $maxdev = 0;
1151
1152     if ($drive->{interface} eq 'virtio') {
1153         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1154         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1155         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1156     } elsif ($drive->{interface} eq 'scsi') {
1157
1158         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1159         my $unit = $drive->{index} % $maxdev;
1160         my $devicetype = 'hd';
1161         my $path = '';
1162         if (drive_is_cdrom($drive)) {
1163               $devicetype = 'cd';
1164         } else {
1165               if ($drive->{file} =~ m|^/|) {
1166                   $path = $drive->{file};
1167               } else {
1168                   $path = PVE::Storage::path($storecfg, $drive->{file});
1169               }
1170
1171               if($path =~ m/^iscsi\:\/\//){
1172                   $devicetype = 'generic';
1173               } else {
1174                   if (my $info = path_is_scsi($path)) {
1175                       if ($info->{type} == 0) {
1176                           $devicetype = 'block';
1177                       } elsif ($info->{type} == 1) { # tape
1178                           $devicetype = 'generic';
1179                       }
1180                   }
1181               }
1182          }
1183
1184         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1185             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1186         } else {
1187             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1188         }
1189
1190     } elsif ($drive->{interface} eq 'ide'){
1191         $maxdev = 2;
1192         my $controller = int($drive->{index} / $maxdev);
1193         my $unit = $drive->{index} % $maxdev;
1194         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1195
1196         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1197     } elsif ($drive->{interface} eq 'sata'){
1198         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1199         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1200         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1201     } elsif ($drive->{interface} eq 'usb') {
1202         die "implement me";
1203         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1204     } else {
1205         die "unsupported interface type";
1206     }
1207
1208     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1209
1210     return $device;
1211 }
1212
1213 sub get_initiator_name {
1214     my $initiator;
1215
1216     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1217     while (defined(my $line = <$fh>)) {
1218         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1219         $initiator = $1;
1220         last;
1221     }
1222     $fh->close();
1223
1224     return $initiator;
1225 }
1226
1227 sub print_drive_full {
1228     my ($storecfg, $vmid, $drive) = @_;
1229
1230     my $opts = '';
1231     foreach my $o (@qemu_drive_options) {
1232         next if $o eq 'bootindex';
1233         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1234     }
1235
1236     foreach my $o (qw(bps bps_rd bps_wr)) {
1237         my $v = $drive->{"m$o"};
1238         $opts .= ",$o=" . int($v*1024*1024) if $v;
1239     }
1240
1241     # aio native works only with O_DIRECT
1242     if (!$drive->{aio}) {
1243         if(!$drive->{cache} || $drive->{cache} eq 'none' || $drive->{cache} eq 'directsync') {
1244             $opts .= ",aio=native";
1245         } else {
1246             $opts .= ",aio=threads";
1247         }
1248     }
1249
1250
1251     my $path;
1252     my $volid = $drive->{file};
1253     if (drive_is_cdrom($drive)) {
1254         $path = get_iso_path($storecfg, $vmid, $volid);
1255     } else {
1256         if ($volid =~ m|^/|) {
1257             $path = $volid;
1258         } else {
1259             $path = PVE::Storage::path($storecfg, $volid);
1260         }
1261     }
1262
1263     $opts .= ",cache=none" if !$drive->{cache} && !drive_is_cdrom($drive);
1264
1265     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1266     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1267
1268     my $pathinfo = $path ? "file=$path," : '';
1269
1270     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1271 }
1272
1273 sub print_netdevice_full {
1274     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1275
1276     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1277
1278     my $device = $net->{model};
1279     if ($net->{model} eq 'virtio') {
1280          $device = 'virtio-net-pci';
1281      };
1282
1283     my $pciaddr = print_pci_addr("$netid", $bridges);
1284     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1285     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1286         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1287         my $vectors = $net->{queues} * 2 + 2;
1288         $tmpstr .= ",vectors=$vectors,mq=on";
1289     }
1290     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1291     return $tmpstr;
1292 }
1293
1294 sub print_netdev_full {
1295     my ($vmid, $conf, $net, $netid) = @_;
1296
1297     my $i = '';
1298     if ($netid =~ m/^net(\d+)$/) {
1299         $i = int($1);
1300     }
1301
1302     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1303
1304     my $ifname = "tap${vmid}i$i";
1305
1306     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1307     die "interface name '$ifname' is too long (max 15 character)\n"
1308         if length($ifname) >= 16;
1309
1310     my $vhostparam = '';
1311     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1312
1313     my $vmname = $conf->{name} || "vm$vmid";
1314
1315     my $netdev = "";
1316
1317     if ($net->{bridge}) {
1318         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1319     } else {
1320         $netdev = "type=user,id=$netid,hostname=$vmname";
1321     }
1322
1323     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1324
1325     return $netdev;
1326 }
1327
1328 sub drive_is_cdrom {
1329     my ($drive) = @_;
1330
1331     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1332
1333 }
1334
1335 sub parse_numa {
1336     my ($data) = @_;
1337
1338     my $res = {};
1339
1340     foreach my $kvp (split(/,/, $data)) {
1341
1342         if ($kvp =~ m/^memory=(\S+)$/) {
1343             $res->{memory} = $1;
1344         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1345             $res->{policy} = $1;
1346         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1347             $res->{cpus}->{start} = $1;
1348             $res->{cpus}->{end} = $3;
1349         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1350             $res->{hostnodes}->{start} = $1;
1351             $res->{hostnodes}->{end} = $3;
1352         } else {
1353             return undef;
1354         }
1355     }
1356
1357     return $res;
1358 }
1359
1360 sub parse_hostpci {
1361     my ($value) = @_;
1362
1363     return undef if !$value;
1364
1365
1366     my @list = split(/,/, $value);
1367     my $found;
1368
1369     my $res = {};
1370     foreach my $kv (@list) {
1371
1372         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1373             $found = 1;
1374             if(defined($4)){
1375                 push @{$res->{pciid}}, { id => $2 , function => $4};
1376
1377             }else{
1378                 my $pcidevices = lspci($2);
1379                 $res->{pciid} = $pcidevices->{$2};
1380             }
1381         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1382             $res->{driver} = $1;
1383         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1384             $res->{rombar} = $1;
1385         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1386             $res->{'x-vga'} = $1;
1387         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1388             $res->{pcie} = 1 if $1 == 1;
1389         } else {
1390             warn "unknown hostpci setting '$kv'\n";
1391         }
1392     }
1393
1394     return undef if !$found;
1395
1396     return $res;
1397 }
1398
1399 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1400 sub parse_net {
1401     my ($data) = @_;
1402
1403     my $res = {};
1404
1405     foreach my $kvp (split(/,/, $data)) {
1406
1407         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1408             my $model = lc($1);
1409             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1410             $res->{model} = $model;
1411             $res->{macaddr} = $mac;
1412         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1413             $res->{bridge} = $1;
1414         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1415             $res->{queues} = $1;
1416         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1417             $res->{rate} = $1;
1418         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1419             $res->{tag} = $1;
1420         } elsif ($kvp =~ m/^firewall=([01])$/) {
1421             $res->{firewall} = $1;
1422         } elsif ($kvp =~ m/^link_down=([01])$/) {
1423             $res->{link_down} = $1;
1424         } else {
1425             return undef;
1426         }
1427
1428     }
1429
1430     return undef if !$res->{model};
1431
1432     return $res;
1433 }
1434
1435 sub print_net {
1436     my $net = shift;
1437
1438     my $res = "$net->{model}";
1439     $res .= "=$net->{macaddr}" if $net->{macaddr};
1440     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1441     $res .= ",rate=$net->{rate}" if $net->{rate};
1442     $res .= ",tag=$net->{tag}" if $net->{tag};
1443     $res .= ",firewall=1" if $net->{firewall};
1444     $res .= ",link_down=1" if $net->{link_down};
1445     $res .= ",queues=$net->{queues}" if $net->{queues};
1446
1447     return $res;
1448 }
1449
1450 sub add_random_macs {
1451     my ($settings) = @_;
1452
1453     foreach my $opt (keys %$settings) {
1454         next if $opt !~ m/^net(\d+)$/;
1455         my $net = parse_net($settings->{$opt});
1456         next if !$net;
1457         $settings->{$opt} = print_net($net);
1458     }
1459 }
1460
1461 sub add_unused_volume {
1462     my ($config, $volid) = @_;
1463
1464     my $key;
1465     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1466         my $test = "unused$ind";
1467         if (my $vid = $config->{$test}) {
1468             return if $vid eq $volid; # do not add duplicates
1469         } else {
1470             $key = $test;
1471         }
1472     }
1473
1474     die "To many unused volume - please delete them first.\n" if !$key;
1475
1476     $config->{$key} = $volid;
1477
1478     return $key;
1479 }
1480
1481 sub vm_is_volid_owner {
1482     my ($storecfg, $vmid, $volid) = @_;
1483
1484     if ($volid !~  m|^/|) {
1485         my ($path, $owner);
1486         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1487         if ($owner && ($owner == $vmid)) {
1488             return 1;
1489         }
1490     }
1491
1492     return undef;
1493 }
1494
1495 sub vmconfig_delete_pending_option {
1496     my ($conf, $key) = @_;
1497
1498     delete $conf->{pending}->{$key};
1499     my $pending_delete_hash = { $key => 1 };
1500     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1501         $pending_delete_hash->{$opt} = 1;
1502     }
1503     $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
1504 }
1505
1506 sub vmconfig_undelete_pending_option {
1507     my ($conf, $key) = @_;
1508
1509     my $pending_delete_hash = {};
1510     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1511         $pending_delete_hash->{$opt} = 1;
1512     }
1513     delete $pending_delete_hash->{$key};
1514
1515     my @keylist = keys %$pending_delete_hash;
1516     if (scalar(@keylist)) {
1517         $conf->{pending}->{delete} = join(',', @keylist);
1518     } else {
1519         delete $conf->{pending}->{delete};
1520     }
1521 }
1522
1523 sub vmconfig_register_unused_drive {
1524     my ($storecfg, $vmid, $conf, $drive) = @_;
1525
1526     if (!drive_is_cdrom($drive)) {
1527         my $volid = $drive->{file};
1528         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1529             add_unused_volume($conf, $volid, $vmid);
1530         }
1531     }
1532 }
1533
1534 sub vmconfig_cleanup_pending {
1535     my ($conf) = @_;
1536
1537     # remove pending changes when nothing changed
1538     my $changes;
1539     foreach my $opt (keys %{$conf->{pending}}) {
1540         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1541             $changes = 1;
1542             delete $conf->{pending}->{$opt};
1543         }
1544     }
1545
1546     # remove delete if option is not set
1547     my $pending_delete_hash = {};
1548     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1549         if (defined($conf->{$opt})) {
1550             $pending_delete_hash->{$opt} = 1;
1551         } else {
1552             $changes = 1;
1553         }
1554     }
1555
1556     my @keylist = keys %$pending_delete_hash;
1557     if (scalar(@keylist)) {
1558         $conf->{pending}->{delete} = join(',', @keylist);
1559     } else {
1560         delete $conf->{pending}->{delete};
1561     }
1562
1563     return $changes;
1564 }
1565
1566 my $valid_smbios1_options = {
1567     manufacturer => '\S+',
1568     product => '\S+',
1569     version => '\S+',
1570     serial => '\S+',
1571     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1572     sku => '\S+',
1573     family => '\S+',
1574 };
1575
1576 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1577 sub parse_smbios1 {
1578     my ($data) = @_;
1579
1580     my $res = {};
1581
1582     foreach my $kvp (split(/,/, $data)) {
1583         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1584         my ($k, $v) = split(/=/, $kvp);
1585         return undef if !defined($k) || !defined($v);
1586         return undef if !$valid_smbios1_options->{$k};
1587         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1588         $res->{$k} = $v;
1589     }
1590
1591     return $res;
1592 }
1593
1594 sub print_smbios1 {
1595     my ($smbios1) = @_;
1596
1597     my $data = '';
1598     foreach my $k (keys %$smbios1) {
1599         next if !defined($smbios1->{$k});
1600         next if !$valid_smbios1_options->{$k};
1601         $data .= ',' if $data;
1602         $data .= "$k=$smbios1->{$k}";
1603     }
1604     return $data;
1605 }
1606
1607 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1608 sub verify_smbios1 {
1609     my ($value, $noerr) = @_;
1610
1611     return $value if parse_smbios1($value);
1612
1613     return undef if $noerr;
1614
1615     die "unable to parse smbios (type 1) options\n";
1616 }
1617
1618 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1619 sub verify_bootdisk {
1620     my ($value, $noerr) = @_;
1621
1622     return $value if valid_drivename($value);
1623
1624     return undef if $noerr;
1625
1626     die "invalid boot disk '$value'\n";
1627 }
1628
1629 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1630 sub verify_numa {
1631     my ($value, $noerr) = @_;
1632
1633     return $value if parse_numa($value);
1634
1635     return undef if $noerr;
1636
1637     die "unable to parse numa options\n";
1638 }
1639
1640 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1641 sub verify_net {
1642     my ($value, $noerr) = @_;
1643
1644     return $value if parse_net($value);
1645
1646     return undef if $noerr;
1647
1648     die "unable to parse network options\n";
1649 }
1650
1651 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1652 sub verify_drive {
1653     my ($value, $noerr) = @_;
1654
1655     return $value if parse_drive(undef, $value);
1656
1657     return undef if $noerr;
1658
1659     die "unable to parse drive options\n";
1660 }
1661
1662 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1663 sub verify_hostpci {
1664     my ($value, $noerr) = @_;
1665
1666     return $value if parse_hostpci($value);
1667
1668     return undef if $noerr;
1669
1670     die "unable to parse pci id\n";
1671 }
1672
1673 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1674 sub verify_watchdog {
1675     my ($value, $noerr) = @_;
1676
1677     return $value if parse_watchdog($value);
1678
1679     return undef if $noerr;
1680
1681     die "unable to parse watchdog options\n";
1682 }
1683
1684 sub parse_watchdog {
1685     my ($value) = @_;
1686
1687     return undef if !$value;
1688
1689     my $res = {};
1690
1691     foreach my $p (split(/,/, $value)) {
1692         next if $p =~ m/^\s*$/;
1693
1694         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1695             $res->{model} = $2;
1696         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1697             $res->{action} = $2;
1698         } else {
1699             return undef;
1700         }
1701     }
1702
1703     return $res;
1704 }
1705
1706 sub parse_usb_device {
1707     my ($value) = @_;
1708
1709     return undef if !$value;
1710
1711     my @dl = split(/,/, $value);
1712     my $found;
1713
1714     my $res = {};
1715     foreach my $v (@dl) {
1716         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1717             $found = 1;
1718             $res->{vendorid} = $2;
1719             $res->{productid} = $4;
1720         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1721             $found = 1;
1722             $res->{hostbus} = $1;
1723             $res->{hostport} = $2;
1724         } elsif ($v =~ m/^spice$/) {
1725             $found = 1;
1726             $res->{spice} = 1;
1727         } else {
1728             return undef;
1729         }
1730     }
1731     return undef if !$found;
1732
1733     return $res;
1734 }
1735
1736 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1737 sub verify_usb_device {
1738     my ($value, $noerr) = @_;
1739
1740     return $value if parse_usb_device($value);
1741
1742     return undef if $noerr;
1743
1744     die "unable to parse usb device\n";
1745 }
1746
1747 # add JSON properties for create and set function
1748 sub json_config_properties {
1749     my $prop = shift;
1750
1751     foreach my $opt (keys %$confdesc) {
1752         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1753         $prop->{$opt} = $confdesc->{$opt};
1754     }
1755
1756     return $prop;
1757 }
1758
1759 sub check_type {
1760     my ($key, $value) = @_;
1761
1762     die "unknown setting '$key'\n" if !$confdesc->{$key};
1763
1764     my $type = $confdesc->{$key}->{type};
1765
1766     if (!defined($value)) {
1767         die "got undefined value\n";
1768     }
1769
1770     if ($value =~ m/[\n\r]/) {
1771         die "property contains a line feed\n";
1772     }
1773
1774     if ($type eq 'boolean') {
1775         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1776         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1777         die "type check ('boolean') failed - got '$value'\n";
1778     } elsif ($type eq 'integer') {
1779         return int($1) if $value =~ m/^(\d+)$/;
1780         die "type check ('integer') failed - got '$value'\n";
1781     } elsif ($type eq 'number') {
1782         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1783         die "type check ('number') failed - got '$value'\n";
1784     } elsif ($type eq 'string') {
1785         if (my $fmt = $confdesc->{$key}->{format}) {
1786             if ($fmt eq 'pve-qm-drive') {
1787                 # special case - we need to pass $key to parse_drive()
1788                 my $drive = parse_drive($key, $value);
1789                 return $value if $drive;
1790                 die "unable to parse drive options\n";
1791             }
1792             PVE::JSONSchema::check_format($fmt, $value);
1793             return $value;
1794         }
1795         $value =~ s/^\"(.*)\"$/$1/;
1796         return $value;
1797     } else {
1798         die "internal error"
1799     }
1800 }
1801
1802 sub lock_config_full {
1803     my ($vmid, $timeout, $code, @param) = @_;
1804
1805     my $filename = config_file_lock($vmid);
1806
1807     my $res = lock_file($filename, $timeout, $code, @param);
1808
1809     die $@ if $@;
1810
1811     return $res;
1812 }
1813
1814 sub lock_config_mode {
1815     my ($vmid, $timeout, $shared, $code, @param) = @_;
1816
1817     my $filename = config_file_lock($vmid);
1818
1819     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1820
1821     die $@ if $@;
1822
1823     return $res;
1824 }
1825
1826 sub lock_config {
1827     my ($vmid, $code, @param) = @_;
1828
1829     return lock_config_full($vmid, 10, $code, @param);
1830 }
1831
1832 sub cfs_config_path {
1833     my ($vmid, $node) = @_;
1834
1835     $node = $nodename if !$node;
1836     return "nodes/$node/qemu-server/$vmid.conf";
1837 }
1838
1839 sub check_iommu_support{
1840     #fixme : need to check IOMMU support
1841     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1842
1843     my $iommu=1;
1844     return $iommu;
1845
1846 }
1847
1848 sub config_file {
1849     my ($vmid, $node) = @_;
1850
1851     my $cfspath = cfs_config_path($vmid, $node);
1852     return "/etc/pve/$cfspath";
1853 }
1854
1855 sub config_file_lock {
1856     my ($vmid) = @_;
1857
1858     return "$lock_dir/lock-$vmid.conf";
1859 }
1860
1861 sub touch_config {
1862     my ($vmid) = @_;
1863
1864     my $conf = config_file($vmid);
1865     utime undef, undef, $conf;
1866 }
1867
1868 sub destroy_vm {
1869     my ($storecfg, $vmid, $keep_empty_config) = @_;
1870
1871     my $conffile = config_file($vmid);
1872
1873     my $conf = load_config($vmid);
1874
1875     check_lock($conf);
1876
1877     # only remove disks owned by this VM
1878     foreach_drive($conf, sub {
1879         my ($ds, $drive) = @_;
1880
1881         return if drive_is_cdrom($drive);
1882
1883         my $volid = $drive->{file};
1884
1885         return if !$volid || $volid =~ m|^/|;
1886
1887         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1888         return if !$path || !$owner || ($owner != $vmid);
1889
1890         PVE::Storage::vdisk_free($storecfg, $volid);
1891     });
1892
1893     if ($keep_empty_config) {
1894         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1895     } else {
1896         unlink $conffile;
1897     }
1898
1899     # also remove unused disk
1900     eval {
1901         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1902
1903         eval {
1904             PVE::Storage::foreach_volid($dl, sub {
1905                 my ($volid, $sid, $volname, $d) = @_;
1906                 PVE::Storage::vdisk_free($storecfg, $volid);
1907             });
1908         };
1909         warn $@ if $@;
1910
1911     };
1912     warn $@ if $@;
1913 }
1914
1915 sub load_config {
1916     my ($vmid, $node) = @_;
1917
1918     my $cfspath = cfs_config_path($vmid, $node);
1919
1920     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1921
1922     die "no such VM ('$vmid')\n" if !defined($conf);
1923
1924     return $conf;
1925 }
1926
1927 sub parse_vm_config {
1928     my ($filename, $raw) = @_;
1929
1930     return undef if !defined($raw);
1931
1932     my $res = {
1933         digest => Digest::SHA::sha1_hex($raw),
1934         snapshots => {},
1935         pending => {},
1936     };
1937
1938     $filename =~ m|/qemu-server/(\d+)\.conf$|
1939         || die "got strange filename '$filename'";
1940
1941     my $vmid = $1;
1942
1943     my $conf = $res;
1944     my $descr = '';
1945     my $section = '';
1946
1947     my @lines = split(/\n/, $raw);
1948     foreach my $line (@lines) {
1949         next if $line =~ m/^\s*$/;
1950
1951         if ($line =~ m/^\[PENDING\]\s*$/i) {
1952             $section = 'pending';
1953             $conf->{description} = $descr if $descr;
1954             $descr = '';
1955             $conf = $res->{$section} = {};
1956             next;
1957
1958         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1959             $section = $1;
1960             $conf->{description} = $descr if $descr;
1961             $descr = '';
1962             $conf = $res->{snapshots}->{$section} = {};
1963             next;
1964         }
1965
1966         if ($line =~ m/^\#(.*)\s*$/) {
1967             $descr .= PVE::Tools::decode_text($1) . "\n";
1968             next;
1969         }
1970
1971         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1972             $descr .= PVE::Tools::decode_text($2);
1973         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1974             $conf->{snapstate} = $1;
1975         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1976             my $key = $1;
1977             my $value = $2;
1978             $conf->{$key} = $value;
1979         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1980             my $value = $1;
1981             if ($section eq 'pending') {
1982                 $conf->{delete} = $value; # we parse this later
1983             } else {
1984                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1985             }
1986         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1987             my $key = $1;
1988             my $value = $2;
1989             eval { $value = check_type($key, $value); };
1990             if ($@) {
1991                 warn "vm $vmid - unable to parse value of '$key' - $@";
1992             } else {
1993                 my $fmt = $confdesc->{$key}->{format};
1994                 if ($fmt && $fmt eq 'pve-qm-drive') {
1995                     my $v = parse_drive($key, $value);
1996                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1997                         $v->{file} = $volid;
1998                         $value = print_drive($vmid, $v);
1999                     } else {
2000                         warn "vm $vmid - unable to parse value of '$key'\n";
2001                         next;
2002                     }
2003                 }
2004
2005                 if ($key eq 'cdrom') {
2006                     $conf->{ide2} = $value;
2007                 } else {
2008                     $conf->{$key} = $value;
2009                 }
2010             }
2011         }
2012     }
2013
2014     $conf->{description} = $descr if $descr;
2015
2016     delete $res->{snapstate}; # just to be sure
2017
2018     return $res;
2019 }
2020
2021 sub write_vm_config {
2022     my ($filename, $conf) = @_;
2023
2024     delete $conf->{snapstate}; # just to be sure
2025
2026     if ($conf->{cdrom}) {
2027         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2028         $conf->{ide2} = $conf->{cdrom};
2029         delete $conf->{cdrom};
2030     }
2031
2032     # we do not use 'smp' any longer
2033     if ($conf->{sockets}) {
2034         delete $conf->{smp};
2035     } elsif ($conf->{smp}) {
2036         $conf->{sockets} = $conf->{smp};
2037         delete $conf->{cores};
2038         delete $conf->{smp};
2039     }
2040
2041     my $used_volids = {};
2042
2043     my $cleanup_config = sub {
2044         my ($cref, $pending, $snapname) = @_;
2045
2046         foreach my $key (keys %$cref) {
2047             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2048                 $key eq 'snapstate' || $key eq 'pending';
2049             my $value = $cref->{$key};
2050             if ($key eq 'delete') {
2051                 die "propertry 'delete' is only allowed in [PENDING]\n"
2052                     if !$pending;
2053                 # fixme: check syntax?
2054                 next;
2055             }
2056             eval { $value = check_type($key, $value); };
2057             die "unable to parse value of '$key' - $@" if $@;
2058
2059             $cref->{$key} = $value;
2060
2061             if (!$snapname && valid_drivename($key)) {
2062                 my $drive = parse_drive($key, $value);
2063                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2064             }
2065         }
2066     };
2067
2068     &$cleanup_config($conf);
2069
2070     &$cleanup_config($conf->{pending}, 1);
2071
2072     foreach my $snapname (keys %{$conf->{snapshots}}) {
2073         die "internal error" if $snapname eq 'pending';
2074         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2075     }
2076
2077     # remove 'unusedX' settings if we re-add a volume
2078     foreach my $key (keys %$conf) {
2079         my $value = $conf->{$key};
2080         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2081             delete $conf->{$key};
2082         }
2083     }
2084
2085     my $generate_raw_config = sub {
2086         my ($conf) = @_;
2087
2088         my $raw = '';
2089
2090         # add description as comment to top of file
2091         my $descr = $conf->{description} || '';
2092         foreach my $cl (split(/\n/, $descr)) {
2093             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2094         }
2095
2096         foreach my $key (sort keys %$conf) {
2097             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2098             $raw .= "$key: $conf->{$key}\n";
2099         }
2100         return $raw;
2101     };
2102
2103     my $raw = &$generate_raw_config($conf);
2104
2105     if (scalar(keys %{$conf->{pending}})){
2106         $raw .= "\n[PENDING]\n";
2107         $raw .= &$generate_raw_config($conf->{pending});
2108     }
2109
2110     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2111         $raw .= "\n[$snapname]\n";
2112         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2113     }
2114
2115     return $raw;
2116 }
2117
2118 sub update_config_nolock {
2119     my ($vmid, $conf, $skiplock) = @_;
2120
2121     check_lock($conf) if !$skiplock;
2122
2123     my $cfspath = cfs_config_path($vmid);
2124
2125     PVE::Cluster::cfs_write_file($cfspath, $conf);
2126 }
2127
2128 sub update_config {
2129     my ($vmid, $conf, $skiplock) = @_;
2130
2131     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2132 }
2133
2134 sub load_defaults {
2135
2136     my $res = {};
2137
2138     # we use static defaults from our JSON schema configuration
2139     foreach my $key (keys %$confdesc) {
2140         if (defined(my $default = $confdesc->{$key}->{default})) {
2141             $res->{$key} = $default;
2142         }
2143     }
2144
2145     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2146     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2147
2148     return $res;
2149 }
2150
2151 sub config_list {
2152     my $vmlist = PVE::Cluster::get_vmlist();
2153     my $res = {};
2154     return $res if !$vmlist || !$vmlist->{ids};
2155     my $ids = $vmlist->{ids};
2156
2157     foreach my $vmid (keys %$ids) {
2158         my $d = $ids->{$vmid};
2159         next if !$d->{node} || $d->{node} ne $nodename;
2160         next if !$d->{type} || $d->{type} ne 'qemu';
2161         $res->{$vmid}->{exists} = 1;
2162     }
2163     return $res;
2164 }
2165
2166 # test if VM uses local resources (to prevent migration)
2167 sub check_local_resources {
2168     my ($conf, $noerr) = @_;
2169
2170     my $loc_res = 0;
2171
2172     $loc_res = 1 if $conf->{hostusb}; # old syntax
2173     $loc_res = 1 if $conf->{hostpci}; # old syntax
2174
2175     foreach my $k (keys %$conf) {
2176         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2177         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2178     }
2179
2180     die "VM uses local resources\n" if $loc_res && !$noerr;
2181
2182     return $loc_res;
2183 }
2184
2185 # check if used storages are available on all nodes (use by migrate)
2186 sub check_storage_availability {
2187     my ($storecfg, $conf, $node) = @_;
2188
2189     foreach_drive($conf, sub {
2190         my ($ds, $drive) = @_;
2191
2192         my $volid = $drive->{file};
2193         return if !$volid;
2194
2195         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2196         return if !$sid;
2197
2198         # check if storage is available on both nodes
2199         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2200         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2201    });
2202 }
2203
2204 # list nodes where all VM images are available (used by has_feature API)
2205 sub shared_nodes {
2206     my ($conf, $storecfg) = @_;
2207
2208     my $nodelist = PVE::Cluster::get_nodelist();
2209     my $nodehash = { map { $_ => 1 } @$nodelist };
2210     my $nodename = PVE::INotify::nodename();
2211
2212     foreach_drive($conf, sub {
2213         my ($ds, $drive) = @_;
2214
2215         my $volid = $drive->{file};
2216         return if !$volid;
2217
2218         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2219         if ($storeid) {
2220             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2221             if ($scfg->{disable}) {
2222                 $nodehash = {};
2223             } elsif (my $avail = $scfg->{nodes}) {
2224                 foreach my $node (keys %$nodehash) {
2225                     delete $nodehash->{$node} if !$avail->{$node};
2226                 }
2227             } elsif (!$scfg->{shared}) {
2228                 foreach my $node (keys %$nodehash) {
2229                     delete $nodehash->{$node} if $node ne $nodename
2230                 }
2231             }
2232         }
2233     });
2234
2235     return $nodehash
2236 }
2237
2238 sub check_lock {
2239     my ($conf) = @_;
2240
2241     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2242 }
2243
2244 sub check_cmdline {
2245     my ($pidfile, $pid) = @_;
2246
2247     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2248     if (defined($fh)) {
2249         my $line = <$fh>;
2250         $fh->close;
2251         return undef if !$line;
2252         my @param = split(/\0/, $line);
2253
2254         my $cmd = $param[0];
2255         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2256
2257         for (my $i = 0; $i < scalar (@param); $i++) {
2258             my $p = $param[$i];
2259             next if !$p;
2260             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2261                 my $p = $param[$i+1];
2262                 return 1 if $p && ($p eq $pidfile);
2263                 return undef;
2264             }
2265         }
2266     }
2267     return undef;
2268 }
2269
2270 sub check_running {
2271     my ($vmid, $nocheck, $node) = @_;
2272
2273     my $filename = config_file($vmid, $node);
2274
2275     die "unable to find configuration file for VM $vmid - no such machine\n"
2276         if !$nocheck && ! -f $filename;
2277
2278     my $pidfile = pidfile_name($vmid);
2279
2280     if (my $fd = IO::File->new("<$pidfile")) {
2281         my $st = stat($fd);
2282         my $line = <$fd>;
2283         close($fd);
2284
2285         my $mtime = $st->mtime;
2286         if ($mtime > time()) {
2287             warn "file '$filename' modified in future\n";
2288         }
2289
2290         if ($line =~ m/^(\d+)$/) {
2291             my $pid = $1;
2292             if (check_cmdline($pidfile, $pid)) {
2293                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2294                     return $pid;
2295                 }
2296             }
2297         }
2298     }
2299
2300     return undef;
2301 }
2302
2303 sub vzlist {
2304
2305     my $vzlist = config_list();
2306
2307     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2308
2309     while (defined(my $de = $fd->read)) {
2310         next if $de !~ m/^(\d+)\.pid$/;
2311         my $vmid = $1;
2312         next if !defined($vzlist->{$vmid});
2313         if (my $pid = check_running($vmid)) {
2314             $vzlist->{$vmid}->{pid} = $pid;
2315         }
2316     }
2317
2318     return $vzlist;
2319 }
2320
2321 sub disksize {
2322     my ($storecfg, $conf) = @_;
2323
2324     my $bootdisk = $conf->{bootdisk};
2325     return undef if !$bootdisk;
2326     return undef if !valid_drivename($bootdisk);
2327
2328     return undef if !$conf->{$bootdisk};
2329
2330     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2331     return undef if !defined($drive);
2332
2333     return undef if drive_is_cdrom($drive);
2334
2335     my $volid = $drive->{file};
2336     return undef if !$volid;
2337
2338     return $drive->{size};
2339 }
2340
2341 my $last_proc_pid_stat;
2342
2343 # get VM status information
2344 # This must be fast and should not block ($full == false)
2345 # We only query KVM using QMP if $full == true (this can be slow)
2346 sub vmstatus {
2347     my ($opt_vmid, $full) = @_;
2348
2349     my $res = {};
2350
2351     my $storecfg = PVE::Storage::config();
2352
2353     my $list = vzlist();
2354     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2355
2356     my $cpucount = $cpuinfo->{cpus} || 1;
2357
2358     foreach my $vmid (keys %$list) {
2359         next if $opt_vmid && ($vmid ne $opt_vmid);
2360
2361         my $cfspath = cfs_config_path($vmid);
2362         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2363
2364         my $d = {};
2365         $d->{pid} = $list->{$vmid}->{pid};
2366
2367         # fixme: better status?
2368         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2369
2370         my $size = disksize($storecfg, $conf);
2371         if (defined($size)) {
2372             $d->{disk} = 0; # no info available
2373             $d->{maxdisk} = $size;
2374         } else {
2375             $d->{disk} = 0;
2376             $d->{maxdisk} = 0;
2377         }
2378
2379         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2380         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2381         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2382
2383         $d->{name} = $conf->{name} || "VM $vmid";
2384         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2385
2386         if ($conf->{balloon}) {
2387             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2388             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2389         }
2390
2391         $d->{uptime} = 0;
2392         $d->{cpu} = 0;
2393         $d->{mem} = 0;
2394
2395         $d->{netout} = 0;
2396         $d->{netin} = 0;
2397
2398         $d->{diskread} = 0;
2399         $d->{diskwrite} = 0;
2400
2401         $d->{template} = is_template($conf);
2402
2403         $res->{$vmid} = $d;
2404     }
2405
2406     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2407     foreach my $dev (keys %$netdev) {
2408         next if $dev !~ m/^tap([1-9]\d*)i/;
2409         my $vmid = $1;
2410         my $d = $res->{$vmid};
2411         next if !$d;
2412
2413         $d->{netout} += $netdev->{$dev}->{receive};
2414         $d->{netin} += $netdev->{$dev}->{transmit};
2415     }
2416
2417     my $ctime = gettimeofday;
2418
2419     foreach my $vmid (keys %$list) {
2420
2421         my $d = $res->{$vmid};
2422         my $pid = $d->{pid};
2423         next if !$pid;
2424
2425         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2426         next if !$pstat; # not running
2427
2428         my $used = $pstat->{utime} + $pstat->{stime};
2429
2430         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2431
2432         if ($pstat->{vsize}) {
2433             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2434         }
2435
2436         my $old = $last_proc_pid_stat->{$pid};
2437         if (!$old) {
2438             $last_proc_pid_stat->{$pid} = {
2439                 time => $ctime,
2440                 used => $used,
2441                 cpu => 0,
2442             };
2443             next;
2444         }
2445
2446         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2447
2448         if ($dtime > 1000) {
2449             my $dutime = $used -  $old->{used};
2450
2451             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2452             $last_proc_pid_stat->{$pid} = {
2453                 time => $ctime,
2454                 used => $used,
2455                 cpu => $d->{cpu},
2456             };
2457         } else {
2458             $d->{cpu} = $old->{cpu};
2459         }
2460     }
2461
2462     return $res if !$full;
2463
2464     my $qmpclient = PVE::QMPClient->new();
2465
2466     my $ballooncb = sub {
2467         my ($vmid, $resp) = @_;
2468
2469         my $info = $resp->{'return'};
2470         return if !$info->{max_mem};
2471
2472         my $d = $res->{$vmid};
2473
2474         # use memory assigned to VM
2475         $d->{maxmem} = $info->{max_mem};
2476         $d->{balloon} = $info->{actual};
2477
2478         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2479             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2480             $d->{freemem} = $info->{free_mem};
2481         }
2482
2483     };
2484
2485     my $blockstatscb = sub {
2486         my ($vmid, $resp) = @_;
2487         my $data = $resp->{'return'} || [];
2488         my $totalrdbytes = 0;
2489         my $totalwrbytes = 0;
2490         for my $blockstat (@$data) {
2491             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2492             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2493         }
2494         $res->{$vmid}->{diskread} = $totalrdbytes;
2495         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2496     };
2497
2498     my $statuscb = sub {
2499         my ($vmid, $resp) = @_;
2500
2501         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2502         # this fails if ballon driver is not loaded, so this must be
2503         # the last commnand (following command are aborted if this fails).
2504         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2505
2506         my $status = 'unknown';
2507         if (!defined($status = $resp->{'return'}->{status})) {
2508             warn "unable to get VM status\n";
2509             return;
2510         }
2511
2512         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2513     };
2514
2515     foreach my $vmid (keys %$list) {
2516         next if $opt_vmid && ($vmid ne $opt_vmid);
2517         next if !$res->{$vmid}->{pid}; # not running
2518         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2519     }
2520
2521     $qmpclient->queue_execute(undef, 1);
2522
2523     foreach my $vmid (keys %$list) {
2524         next if $opt_vmid && ($vmid ne $opt_vmid);
2525         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2526     }
2527
2528     return $res;
2529 }
2530
2531 sub foreach_dimm {
2532     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2533
2534     my $dimm_id = 0;
2535     my $current_size = 1024;
2536     my $dimm_size = 512;
2537     return if $current_size == $memory;
2538
2539     for (my $j = 0; $j < 8; $j++) {
2540         for (my $i = 0; $i < 32; $i++) {
2541             my $name = "dimm${dimm_id}";
2542             $dimm_id++;
2543             my $numanode = $i % $sockets;
2544             $current_size += $dimm_size;
2545             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2546             return  $current_size if $current_size >= $memory;
2547         }
2548         $dimm_size *= 2;
2549     }
2550 }
2551
2552 sub foreach_drive {
2553     my ($conf, $func) = @_;
2554
2555     foreach my $ds (keys %$conf) {
2556         next if !valid_drivename($ds);
2557
2558         my $drive = parse_drive($ds, $conf->{$ds});
2559         next if !$drive;
2560
2561         &$func($ds, $drive);
2562     }
2563 }
2564
2565 sub foreach_volid {
2566     my ($conf, $func) = @_;
2567
2568     my $volhash = {};
2569
2570     my $test_volid = sub {
2571         my ($volid, $is_cdrom) = @_;
2572
2573         return if !$volid;
2574
2575         $volhash->{$volid} = $is_cdrom || 0;
2576     };
2577
2578     foreach_drive($conf, sub {
2579         my ($ds, $drive) = @_;
2580         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2581     });
2582
2583     foreach my $snapname (keys %{$conf->{snapshots}}) {
2584         my $snap = $conf->{snapshots}->{$snapname};
2585         &$test_volid($snap->{vmstate}, 0);
2586         foreach_drive($snap, sub {
2587             my ($ds, $drive) = @_;
2588             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2589         });
2590     }
2591
2592     foreach my $volid (keys %$volhash) {
2593         &$func($volid, $volhash->{$volid});
2594     }
2595 }
2596
2597 sub vga_conf_has_spice {
2598     my ($vga) = @_;
2599
2600     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2601
2602     return $1 || 1;
2603 }
2604
2605 sub config_to_command {
2606     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2607
2608     my $cmd = [];
2609     my $globalFlags = [];
2610     my $machineFlags = [];
2611     my $rtcFlags = [];
2612     my $cpuFlags = [];
2613     my $devices = [];
2614     my $pciaddr = '';
2615     my $bridges = {};
2616     my $kvmver = kvm_user_version();
2617     my $vernum = 0; # unknown
2618     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2619         $vernum = $1*1000000+$2*1000;
2620     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2621         $vernum = $1*1000000+$2*1000+$3;
2622     }
2623
2624     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2625
2626     my $have_ovz = -f '/proc/vz/vestat';
2627
2628     my $q35 = machine_type_is_q35($conf);
2629     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2630     my $machine_type = $forcemachine || $conf->{machine};
2631
2632     my $cpuunits = defined($conf->{cpuunits}) ?
2633             $conf->{cpuunits} : $defaults->{cpuunits};
2634
2635     push @$cmd, '/usr/bin/systemd-run';
2636     push @$cmd, '--scope';
2637     push @$cmd, '--slice', "qemu";
2638     push @$cmd, '--unit', $vmid;
2639     push @$cmd, '-p', "CPUShares=$cpuunits";
2640
2641     push @$cmd, '/usr/bin/kvm';
2642
2643     push @$cmd, '-id', $vmid;
2644
2645     my $use_virtio = 0;
2646
2647     my $qmpsocket = qmp_socket($vmid);
2648     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2649     push @$cmd, '-mon', "chardev=qmp,mode=control";
2650
2651     my $socket = vnc_socket($vmid);
2652     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2653
2654     push @$cmd, '-pidfile' , pidfile_name($vmid);
2655
2656     push @$cmd, '-daemonize';
2657
2658     if ($conf->{smbios1}) {
2659         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2660     }
2661
2662     if ($q35) {
2663         # the q35 chipset support native usb2, so we enable usb controller
2664         # by default for this machine type
2665         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2666     } else {
2667         $pciaddr = print_pci_addr("piix3", $bridges);
2668         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2669
2670         my $use_usb2 = 0;
2671         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2672             next if !$conf->{"usb$i"};
2673             $use_usb2 = 1;
2674         }
2675         # include usb device config
2676         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2677     }
2678
2679     my $vga = $conf->{vga};
2680
2681     my $qxlnum = vga_conf_has_spice($vga);
2682     $vga = 'qxl' if $qxlnum;
2683
2684     if (!$vga) {
2685         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2686                                 $conf->{ostype} eq 'win7' ||
2687                                 $conf->{ostype} eq 'w2k8')) {
2688             $vga = 'std';
2689         } else {
2690             $vga = 'cirrus';
2691         }
2692     }
2693
2694     # enable absolute mouse coordinates (needed by vnc)
2695     my $tablet;
2696     if (defined($conf->{tablet})) {
2697         $tablet = $conf->{tablet};
2698     } else {
2699         $tablet = $defaults->{tablet};
2700         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2701         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2702     }
2703
2704     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2705
2706     # host pci devices
2707     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2708         my $d = parse_hostpci($conf->{"hostpci$i"});
2709         next if !$d;
2710
2711         my $pcie = $d->{pcie};
2712         if($pcie){
2713             die "q35 machine model is not enabled" if !$q35;
2714             $pciaddr = print_pcie_addr("hostpci$i");
2715         }else{
2716             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2717         }
2718
2719         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2720         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2721         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2722         if ($xvga && $xvga ne '') {
2723             push @$cpuFlags, 'kvm=off';
2724             $vga = 'none';
2725         }
2726         $driver = "vfio-pci" if $xvga ne '';
2727         my $pcidevices = $d->{pciid};
2728         my $multifunction = 1 if @$pcidevices > 1;
2729
2730         my $j=0;
2731         foreach my $pcidevice (@$pcidevices) {
2732
2733             my $id = "hostpci$i";
2734             $id .= ".$j" if $multifunction;
2735             my $addr = $pciaddr;
2736             $addr .= ".$j" if $multifunction;
2737             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2738
2739             if($j == 0){
2740                 $devicestr .= "$rombar$xvga";
2741                 $devicestr .= ",multifunction=on" if $multifunction;
2742             }
2743
2744             push @$devices, '-device', $devicestr;
2745             $j++;
2746         }
2747     }
2748
2749     # usb devices
2750     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2751         my $d = parse_usb_device($conf->{"usb$i"});
2752         next if !$d;
2753         if ($d->{vendorid} && $d->{productid}) {
2754             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2755         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2756             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2757         } elsif ($d->{spice}) {
2758             # usb redir support for spice
2759             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2760             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2761         }
2762     }
2763
2764     # serial devices
2765     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2766         if (my $path = $conf->{"serial$i"}) {
2767             if ($path eq 'socket') {
2768                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2769                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2770                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2771             } else {
2772                 die "no such serial device\n" if ! -c $path;
2773                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2774                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2775             }
2776         }
2777     }
2778
2779     # parallel devices
2780     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2781         if (my $path = $conf->{"parallel$i"}) {
2782             die "no such parallel device\n" if ! -c $path;
2783             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2784             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2785             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2786         }
2787     }
2788
2789     my $vmname = $conf->{name} || "vm$vmid";
2790
2791     push @$cmd, '-name', $vmname;
2792
2793     my $sockets = 1;
2794     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2795     $sockets = $conf->{sockets} if  $conf->{sockets};
2796
2797     my $cores = $conf->{cores} || 1;
2798
2799     my $maxcpus = $sockets * $cores;
2800
2801     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2802
2803     my $allowed_vcpus = $cpuinfo->{cpus};
2804
2805     die "MAX $maxcpus vcpus allowed per VM on this node\n"
2806         if ($allowed_vcpus < $maxcpus);
2807
2808     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2809
2810     push @$cmd, '-nodefaults';
2811
2812     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2813
2814     my $bootindex_hash = {};
2815     my $i = 1;
2816     foreach my $o (split(//, $bootorder)) {
2817         $bootindex_hash->{$o} = $i*100;
2818         $i++;
2819     }
2820
2821     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2822
2823     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2824
2825     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2826
2827     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2828
2829     # time drift fix
2830     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2831
2832     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2833     my $useLocaltime = $conf->{localtime};
2834
2835     if (my $ost = $conf->{ostype}) {
2836         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2837
2838         if ($ost =~ m/^w/) { # windows
2839             $useLocaltime = 1 if !defined($conf->{localtime});
2840
2841             # use time drift fix when acpi is enabled
2842             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2843                 $tdf = 1 if !defined($conf->{tdf});
2844             }
2845         }
2846
2847         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2848             $ost eq 'wvista') {
2849             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2850             push @$cmd, '-no-hpet';
2851             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2852                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2853                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2854                 push @$cpuFlags , 'hv_time' if !$nokvm;
2855
2856             } else {
2857                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2858             }
2859         }
2860
2861         if ($ost eq 'win7' || $ost eq 'win8') {
2862             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2863         }
2864     }
2865
2866     push @$rtcFlags, 'driftfix=slew' if $tdf;
2867
2868     if ($nokvm) {
2869         push @$machineFlags, 'accel=tcg';
2870     } else {
2871         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2872     }
2873
2874     if ($machine_type) {
2875         push @$machineFlags, "type=${machine_type}";
2876     }
2877
2878     if ($conf->{startdate}) {
2879         push @$rtcFlags, "base=$conf->{startdate}";
2880     } elsif ($useLocaltime) {
2881         push @$rtcFlags, 'base=localtime';
2882     }
2883
2884     my $cpu = $nokvm ? "qemu64" : "kvm64";
2885     $cpu = $conf->{cpu} if $conf->{cpu};
2886
2887     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2888
2889     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2890
2891     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2892
2893     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2894
2895     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2896
2897         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2898         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2899     }
2900
2901     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2902
2903     push @$cmd, '-cpu', "$cpu,enforce";
2904
2905     my $memory = $conf->{memory} || $defaults->{memory};
2906     my $static_memory = 0;
2907     my $dimm_memory = 0;
2908
2909     if ($hotplug_features->{memory}) {
2910         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2911         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2912         $static_memory = $STATICMEM;
2913         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2914         $dimm_memory = $memory - $static_memory;
2915         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2916
2917     } else {
2918
2919         $static_memory = $memory;
2920         push @$cmd, '-m', $static_memory;
2921     }
2922
2923     if ($conf->{numa}) {
2924
2925         my $numa_totalmemory = undef;
2926         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2927             next if !$conf->{"numa$i"};
2928             my $numa = parse_numa($conf->{"numa$i"});
2929             next if !$numa;
2930             # memory
2931             die "missing numa node$i memory value\n" if !$numa->{memory};
2932             my $numa_memory = $numa->{memory};
2933             $numa_totalmemory += $numa_memory;
2934             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2935
2936             # cpus
2937             my $cpus_start = $numa->{cpus}->{start};
2938             die "missing numa node$i cpus\n" if !defined($cpus_start);
2939             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2940             my $cpus = $cpus_start;
2941             if (defined($cpus_end)) {
2942                 $cpus .= "-$cpus_end";
2943                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2944             }
2945
2946             # hostnodes
2947             my $hostnodes_start = $numa->{hostnodes}->{start};
2948             if (defined($hostnodes_start)) {
2949                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2950                 my $hostnodes = $hostnodes_start;
2951                 if (defined($hostnodes_end)) {
2952                     $hostnodes .= "-$hostnodes_end";
2953                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2954                 }
2955
2956                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2957                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2958                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2959                 }
2960
2961                 # policy
2962                 my $policy = $numa->{policy};
2963                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2964                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2965             }
2966
2967             push @$cmd, '-object', $numa_object;
2968             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2969         }
2970
2971         die "total memory for NUMA nodes must be equal to vm static memory\n"
2972             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2973
2974         #if no custom tology, we split memory and cores across numa nodes
2975         if(!$numa_totalmemory) {
2976
2977             my $numa_memory = ($static_memory / $sockets) . "M";
2978
2979             for (my $i = 0; $i < $sockets; $i++)  {
2980
2981                 my $cpustart = ($cores * $i);
2982                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2983                 my $cpus = $cpustart;
2984                 $cpus .= "-$cpuend" if $cpuend;
2985
2986                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2987                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2988             }
2989         }
2990     }
2991
2992     if ($hotplug_features->{memory}) {
2993         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
2994             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
2995             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
2996             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
2997
2998             #if dimm_memory is not aligned to dimm map
2999             if($current_size > $memory) {
3000                  $conf->{memory} = $current_size;
3001                  update_config_nolock($vmid, $conf, 1);
3002             }
3003         });
3004     }
3005
3006     push @$cmd, '-S' if $conf->{freeze};
3007
3008     # set keyboard layout
3009     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3010     push @$cmd, '-k', $kb if $kb;
3011
3012     # enable sound
3013     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3014     #push @$cmd, '-soundhw', 'es1370';
3015     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3016
3017     if($conf->{agent}) {
3018         my $qgasocket = qmp_socket($vmid, 1);
3019         my $pciaddr = print_pci_addr("qga0", $bridges);
3020         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3021         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3022         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3023     }
3024
3025     my $spice_port;
3026
3027     if ($qxlnum) {
3028         if ($qxlnum > 1) {
3029             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3030                 for(my $i = 1; $i < $qxlnum; $i++){
3031                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3032                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3033                 }
3034             } else {
3035                 # assume other OS works like Linux
3036                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3037                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3038             }
3039         }
3040
3041         my $pciaddr = print_pci_addr("spice", $bridges);
3042
3043         my $nodename = PVE::INotify::nodename();
3044         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3045         $spice_port = PVE::Tools::next_spice_port($pfamily);
3046
3047         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3048
3049         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3050         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3051         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3052     }
3053
3054     # enable balloon by default, unless explicitly disabled
3055     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3056         $pciaddr = print_pci_addr("balloon0", $bridges);
3057         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3058     }
3059
3060     if ($conf->{watchdog}) {
3061         my $wdopts = parse_watchdog($conf->{watchdog});
3062         $pciaddr = print_pci_addr("watchdog", $bridges);
3063         my $watchdog = $wdopts->{model} || 'i6300esb';
3064         push @$devices, '-device', "$watchdog$pciaddr";
3065         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3066     }
3067
3068     my $vollist = [];
3069     my $scsicontroller = {};
3070     my $ahcicontroller = {};
3071     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3072
3073     # Add iscsi initiator name if available
3074     if (my $initiator = get_initiator_name()) {
3075         push @$devices, '-iscsi', "initiator-name=$initiator";
3076     }
3077
3078     foreach_drive($conf, sub {
3079         my ($ds, $drive) = @_;
3080
3081         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3082             push @$vollist, $drive->{file};
3083         }
3084
3085         $use_virtio = 1 if $ds =~ m/^virtio/;
3086
3087         if (drive_is_cdrom ($drive)) {
3088             if ($bootindex_hash->{d}) {
3089                 $drive->{bootindex} = $bootindex_hash->{d};
3090                 $bootindex_hash->{d} += 1;
3091             }
3092         } else {
3093             if ($bootindex_hash->{c}) {
3094                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3095                 $bootindex_hash->{c} += 1;
3096             }
3097         }
3098
3099         if($drive->{interface} eq 'virtio'){
3100            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3101         }
3102
3103         if ($drive->{interface} eq 'scsi') {
3104
3105             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3106
3107             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3108             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3109
3110             my $iothread = '';
3111             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3112                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3113                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3114             }
3115
3116             my $queues = '';
3117             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3118                 $queues = ",num_queues=$drive->{queues}";
3119             }
3120
3121             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3122             $scsicontroller->{$controller}=1;
3123         }
3124
3125         if ($drive->{interface} eq 'sata') {
3126            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3127            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3128            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3129            $ahcicontroller->{$controller}=1;
3130         }
3131
3132         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3133         push @$devices, '-drive',$drive_cmd;
3134         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3135     });
3136
3137     for (my $i = 0; $i < $MAX_NETS; $i++) {
3138          next if !$conf->{"net$i"};
3139          my $d = parse_net($conf->{"net$i"});
3140          next if !$d;
3141
3142          $use_virtio = 1 if $d->{model} eq 'virtio';
3143
3144          if ($bootindex_hash->{n}) {
3145             $d->{bootindex} = $bootindex_hash->{n};
3146             $bootindex_hash->{n} += 1;
3147          }
3148
3149          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3150          push @$devices, '-netdev', $netdevfull;
3151
3152          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3153          push @$devices, '-device', $netdevicefull;
3154     }
3155
3156     if (!$q35) {
3157         # add pci bridges
3158         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3159            $bridges->{1} = 1;
3160            $bridges->{2} = 1;
3161         }
3162
3163         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3164
3165         while (my ($k, $v) = each %$bridges) {
3166             $pciaddr = print_pci_addr("pci.$k");
3167             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3168         }
3169     }
3170
3171     # hack: virtio with fairsched is unreliable, so we do not use fairsched
3172     # when the VM uses virtio devices.
3173     if (!$use_virtio && $have_ovz) {
3174
3175         my $cpuunits = defined($conf->{cpuunits}) ?
3176             $conf->{cpuunits} : $defaults->{cpuunits};
3177
3178         push @$cmd, '-cpuunits', $cpuunits if $cpuunits;
3179
3180         # fixme: cpulimit is currently ignored
3181         #push @$cmd, '-cpulimit', $conf->{cpulimit} if $conf->{cpulimit};
3182     }
3183
3184     # add custom args
3185     if ($conf->{args}) {
3186         my $aa = PVE::Tools::split_args($conf->{args});
3187         push @$cmd, @$aa;
3188     }
3189
3190     push @$cmd, @$devices;
3191     push @$cmd, '-rtc', join(',', @$rtcFlags)
3192         if scalar(@$rtcFlags);
3193     push @$cmd, '-machine', join(',', @$machineFlags)
3194         if scalar(@$machineFlags);
3195     push @$cmd, '-global', join(',', @$globalFlags)
3196         if scalar(@$globalFlags);
3197
3198     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3199 }
3200
3201 sub vnc_socket {
3202     my ($vmid) = @_;
3203     return "${var_run_tmpdir}/$vmid.vnc";
3204 }
3205
3206 sub spice_port {
3207     my ($vmid) = @_;
3208
3209     my $res = vm_mon_cmd($vmid, 'query-spice');
3210
3211     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3212 }
3213
3214 sub qmp_socket {
3215     my ($vmid, $qga) = @_;
3216     my $sockettype = $qga ? 'qga' : 'qmp';
3217     return "${var_run_tmpdir}/$vmid.$sockettype";
3218 }
3219
3220 sub pidfile_name {
3221     my ($vmid) = @_;
3222     return "${var_run_tmpdir}/$vmid.pid";
3223 }
3224
3225 sub vm_devices_list {
3226     my ($vmid) = @_;
3227
3228     my $res = vm_mon_cmd($vmid, 'query-pci');
3229     my $devices = {};
3230     foreach my $pcibus (@$res) {
3231         foreach my $device (@{$pcibus->{devices}}) {
3232             next if !$device->{'qdev_id'};
3233             if ($device->{'pci_bridge'}) {
3234                 $devices->{$device->{'qdev_id'}} = 1;
3235                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3236                     next if !$bridge_device->{'qdev_id'};
3237                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3238                     $devices->{$device->{'qdev_id'}}++;
3239                 }
3240             } else {
3241                 $devices->{$device->{'qdev_id'}} = 1;
3242             }
3243         }
3244     }
3245
3246     my $resblock = vm_mon_cmd($vmid, 'query-block');
3247     foreach my $block (@$resblock) {
3248         if($block->{device} =~ m/^drive-(\S+)/){
3249                 $devices->{$1} = 1;
3250         }
3251     }
3252
3253     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3254     foreach my $mice (@$resmice) {
3255         if ($mice->{name} eq 'QEMU HID Tablet') {
3256             $devices->{tablet} = 1;
3257             last;
3258         }
3259     }
3260
3261     return $devices;
3262 }
3263
3264 sub vm_deviceplug {
3265     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3266
3267     my $q35 = machine_type_is_q35($conf);
3268
3269     my $devices_list = vm_devices_list($vmid);
3270     return 1 if defined($devices_list->{$deviceid});
3271
3272     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3273
3274     if ($deviceid eq 'tablet') {
3275
3276         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3277
3278     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3279
3280         qemu_iothread_add($vmid, $deviceid, $device);
3281
3282         qemu_driveadd($storecfg, $vmid, $device);
3283         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3284
3285         qemu_deviceadd($vmid, $devicefull);
3286         eval { qemu_deviceaddverify($vmid, $deviceid); };
3287         if (my $err = $@) {
3288             eval { qemu_drivedel($vmid, $deviceid); };
3289             warn $@ if $@;
3290             die $err;
3291         }
3292
3293     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3294
3295
3296         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3297         my $pciaddr = print_pci_addr($deviceid);
3298         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3299
3300         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3301
3302         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3303             qemu_iothread_add($vmid, $deviceid, $device);
3304             $devicefull .= ",iothread=iothread-$deviceid";
3305         }
3306
3307         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3308             $devicefull .= ",num_queues=$device->{queues}";
3309         }
3310
3311         qemu_deviceadd($vmid, $devicefull);
3312         qemu_deviceaddverify($vmid, $deviceid);
3313
3314     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3315
3316         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3317         qemu_driveadd($storecfg, $vmid, $device);
3318
3319         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3320         eval { qemu_deviceadd($vmid, $devicefull); };
3321         if (my $err = $@) {
3322             eval { qemu_drivedel($vmid, $deviceid); };
3323             warn $@ if $@;
3324             die $err;
3325         }
3326
3327     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3328
3329         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3330         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3331         qemu_deviceadd($vmid, $netdevicefull);
3332         eval { qemu_deviceaddverify($vmid, $deviceid); };
3333         if (my $err = $@) {
3334             eval { qemu_netdevdel($vmid, $deviceid); };
3335             warn $@ if $@;
3336             die $err;
3337         }
3338
3339     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3340
3341         my $bridgeid = $2;
3342         my $pciaddr = print_pci_addr($deviceid);
3343         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3344
3345         qemu_deviceadd($vmid, $devicefull);
3346         qemu_deviceaddverify($vmid, $deviceid);
3347
3348     } else {
3349         die "can't hotplug device '$deviceid'\n";
3350     }
3351
3352     return 1;
3353 }
3354
3355 # fixme: this should raise exceptions on error!
3356 sub vm_deviceunplug {
3357     my ($vmid, $conf, $deviceid) = @_;
3358
3359     my $devices_list = vm_devices_list($vmid);
3360     return 1 if !defined($devices_list->{$deviceid});
3361
3362     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3363
3364     if ($deviceid eq 'tablet') {
3365
3366         qemu_devicedel($vmid, $deviceid);
3367
3368     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3369
3370         qemu_devicedel($vmid, $deviceid);
3371         qemu_devicedelverify($vmid, $deviceid);
3372         qemu_drivedel($vmid, $deviceid);
3373         qemu_iothread_del($conf, $vmid, $deviceid);
3374
3375     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3376
3377         qemu_devicedel($vmid, $deviceid);
3378         qemu_devicedelverify($vmid, $deviceid);
3379         qemu_iothread_del($conf, $vmid, $deviceid);
3380
3381     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3382
3383         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3384         my $device = parse_drive($deviceid, $conf->{$deviceid});
3385         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3386
3387         qemu_devicedel($vmid, $deviceid);
3388         qemu_drivedel($vmid, $deviceid);
3389         qemu_deletescsihw($conf, $vmid, $deviceid);
3390
3391     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3392
3393         qemu_devicedel($vmid, $deviceid);
3394         qemu_devicedelverify($vmid, $deviceid);
3395         qemu_netdevdel($vmid, $deviceid);
3396
3397     } else {
3398         die "can't unplug device '$deviceid'\n";
3399     }
3400
3401     return 1;
3402 }
3403
3404 sub qemu_deviceadd {
3405     my ($vmid, $devicefull) = @_;
3406
3407     $devicefull = "driver=".$devicefull;
3408     my %options =  split(/[=,]/, $devicefull);
3409
3410     vm_mon_cmd($vmid, "device_add" , %options);
3411 }
3412
3413 sub qemu_devicedel {
3414     my ($vmid, $deviceid) = @_;
3415
3416     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3417 }
3418
3419 sub qemu_iothread_add {
3420     my($vmid, $deviceid, $device) = @_;
3421
3422     if ($device->{iothread}) {
3423         my $iothreads = vm_iothreads_list($vmid);
3424         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3425     }
3426 }
3427
3428 sub qemu_iothread_del {
3429     my($conf, $vmid, $deviceid) = @_;
3430
3431     my $device = parse_drive($deviceid, $conf->{$deviceid});
3432     if ($device->{iothread}) {
3433         my $iothreads = vm_iothreads_list($vmid);
3434         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3435     }
3436 }
3437
3438 sub qemu_objectadd {
3439     my($vmid, $objectid, $qomtype) = @_;
3440
3441     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3442
3443     return 1;
3444 }
3445
3446 sub qemu_objectdel {
3447     my($vmid, $objectid) = @_;
3448
3449     vm_mon_cmd($vmid, "object-del", id => $objectid);
3450
3451     return 1;
3452 }
3453
3454 sub qemu_driveadd {
3455     my ($storecfg, $vmid, $device) = @_;
3456
3457     my $drive = print_drive_full($storecfg, $vmid, $device);
3458     $drive =~ s/\\/\\\\/g;
3459     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3460
3461     # If the command succeeds qemu prints: "OK"
3462     return 1 if $ret =~ m/OK/s;
3463
3464     die "adding drive failed: $ret\n";
3465 }
3466
3467 sub qemu_drivedel {
3468     my($vmid, $deviceid) = @_;
3469
3470     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3471     $ret =~ s/^\s+//;
3472
3473     return 1 if $ret eq "";
3474
3475     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3476     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3477
3478     die "deleting drive $deviceid failed : $ret\n";
3479 }
3480
3481 sub qemu_deviceaddverify {
3482     my ($vmid, $deviceid) = @_;
3483
3484     for (my $i = 0; $i <= 5; $i++) {
3485          my $devices_list = vm_devices_list($vmid);
3486          return 1 if defined($devices_list->{$deviceid});
3487          sleep 1;
3488     }
3489
3490     die "error on hotplug device '$deviceid'\n";
3491 }
3492
3493
3494 sub qemu_devicedelverify {
3495     my ($vmid, $deviceid) = @_;
3496
3497     # need to verify that the device is correctly removed as device_del
3498     # is async and empty return is not reliable
3499
3500     for (my $i = 0; $i <= 5; $i++) {
3501          my $devices_list = vm_devices_list($vmid);
3502          return 1 if !defined($devices_list->{$deviceid});
3503          sleep 1;
3504     }
3505
3506     die "error on hot-unplugging device '$deviceid'\n";
3507 }
3508
3509 sub qemu_findorcreatescsihw {
3510     my ($storecfg, $conf, $vmid, $device) = @_;
3511
3512     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3513
3514     my $scsihwid="$controller_prefix$controller";
3515     my $devices_list = vm_devices_list($vmid);
3516
3517     if(!defined($devices_list->{$scsihwid})) {
3518         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3519     }
3520
3521     return 1;
3522 }
3523
3524 sub qemu_deletescsihw {
3525     my ($conf, $vmid, $opt) = @_;
3526
3527     my $device = parse_drive($opt, $conf->{$opt});
3528
3529     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3530         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3531         return 1;
3532     }
3533
3534     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3535
3536     my $devices_list = vm_devices_list($vmid);
3537     foreach my $opt (keys %{$devices_list}) {
3538         if (PVE::QemuServer::valid_drivename($opt)) {
3539             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3540             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3541                 return 1;
3542             }
3543         }
3544     }
3545
3546     my $scsihwid="scsihw$controller";
3547
3548     vm_deviceunplug($vmid, $conf, $scsihwid);
3549
3550     return 1;
3551 }
3552
3553 sub qemu_add_pci_bridge {
3554     my ($storecfg, $conf, $vmid, $device) = @_;
3555
3556     my $bridges = {};
3557
3558     my $bridgeid;
3559
3560     print_pci_addr($device, $bridges);
3561
3562     while (my ($k, $v) = each %$bridges) {
3563         $bridgeid = $k;
3564     }
3565     return 1 if !defined($bridgeid) || $bridgeid < 1;
3566
3567     my $bridge = "pci.$bridgeid";
3568     my $devices_list = vm_devices_list($vmid);
3569
3570     if (!defined($devices_list->{$bridge})) {
3571         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3572     }
3573
3574     return 1;
3575 }
3576
3577 sub qemu_set_link_status {
3578     my ($vmid, $device, $up) = @_;
3579
3580     vm_mon_cmd($vmid, "set_link", name => $device,
3581                up => $up ? JSON::true : JSON::false);
3582 }
3583
3584 sub qemu_netdevadd {
3585     my ($vmid, $conf, $device, $deviceid) = @_;
3586
3587     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3588     my %options =  split(/[=,]/, $netdev);
3589
3590     vm_mon_cmd($vmid, "netdev_add",  %options);
3591     return 1;
3592 }
3593
3594 sub qemu_netdevdel {
3595     my ($vmid, $deviceid) = @_;
3596
3597     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3598 }
3599
3600 sub qemu_cpu_hotplug {
3601     my ($vmid, $conf, $vcpus) = @_;
3602
3603     my $sockets = 1;
3604     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3605     $sockets = $conf->{sockets} if  $conf->{sockets};
3606     my $cores = $conf->{cores} || 1;
3607     my $maxcpus = $sockets * $cores;
3608
3609     $vcpus = $maxcpus if !$vcpus;
3610
3611     die "you can't add more vcpus than maxcpus\n"
3612         if $vcpus > $maxcpus;
3613
3614     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3615     die "online cpu unplug is not yet possible\n"
3616         if $vcpus < $currentvcpus;
3617
3618     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3619     die "vcpus in running vm is different than configuration\n"
3620         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3621
3622     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3623         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3624     }
3625 }
3626
3627 sub qemu_memory_hotplug {
3628     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3629
3630     return $value if !check_running($vmid);
3631
3632     my $memory = $conf->{memory} || $defaults->{memory};
3633     $value = $defaults->{memory} if !$value;
3634     return $value if $value == $memory;
3635
3636     my $static_memory = $STATICMEM;
3637     my $dimm_memory = $memory - $static_memory;
3638
3639     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3640     die "memory unplug is not yet available" if $value < $memory;
3641     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3642
3643
3644     my $sockets = 1;
3645     $sockets = $conf->{sockets} if $conf->{sockets};
3646
3647     foreach_dimm($conf, $vmid, $value, $sockets, sub {
3648         my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3649
3650             return if $current_size <= $conf->{memory};
3651
3652             eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3653             if (my $err = $@) {
3654                 eval { qemu_objectdel($vmid, "mem-$name"); };
3655                 die $err;
3656             }
3657
3658             eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3659             if (my $err = $@) {
3660                 eval { qemu_objectdel($vmid, "mem-$name"); };
3661                 die $err;
3662             }
3663             #update conf after each succesful module hotplug
3664             $conf->{memory} = $current_size;
3665             update_config_nolock($vmid, $conf, 1);
3666     });
3667 }
3668
3669 sub qemu_block_set_io_throttle {
3670     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3671
3672     return if !check_running($vmid) ;
3673
3674     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3675
3676 }
3677
3678 # old code, only used to shutdown old VM after update
3679 sub __read_avail {
3680     my ($fh, $timeout) = @_;
3681
3682     my $sel = new IO::Select;
3683     $sel->add($fh);
3684
3685     my $res = '';
3686     my $buf;
3687
3688     my @ready;
3689     while (scalar (@ready = $sel->can_read($timeout))) {
3690         my $count;
3691         if ($count = $fh->sysread($buf, 8192)) {
3692             if ($buf =~ /^(.*)\(qemu\) $/s) {
3693                 $res .= $1;
3694                 last;
3695             } else {
3696                 $res .= $buf;
3697             }
3698         } else {
3699             if (!defined($count)) {
3700                 die "$!\n";
3701             }
3702             last;
3703         }
3704     }
3705
3706     die "monitor read timeout\n" if !scalar(@ready);
3707
3708     return $res;
3709 }
3710
3711 # old code, only used to shutdown old VM after update
3712 sub vm_monitor_command {
3713     my ($vmid, $cmdstr, $nocheck) = @_;
3714
3715     my $res;
3716
3717     eval {
3718         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3719
3720         my $sname = "${var_run_tmpdir}/$vmid.mon";
3721
3722         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3723             die "unable to connect to VM $vmid socket - $!\n";
3724
3725         my $timeout = 3;
3726
3727         # hack: migrate sometime blocks the monitor (when migrate_downtime
3728         # is set)
3729         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3730             $timeout = 60*60; # 1 hour
3731         }
3732
3733         # read banner;
3734         my $data = __read_avail($sock, $timeout);
3735
3736         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3737             die "got unexpected qemu monitor banner\n";
3738         }
3739
3740         my $sel = new IO::Select;
3741         $sel->add($sock);
3742
3743         if (!scalar(my @ready = $sel->can_write($timeout))) {
3744             die "monitor write error - timeout";
3745         }
3746
3747         my $fullcmd = "$cmdstr\r";
3748
3749         # syslog('info', "VM $vmid monitor command: $cmdstr");
3750
3751         my $b;
3752         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3753             die "monitor write error - $!";
3754         }
3755
3756         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3757
3758         $timeout = 20;
3759
3760         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3761             $timeout = 60*60; # 1 hour
3762         } elsif ($cmdstr =~ m/^(eject|change)/) {
3763             $timeout = 60; # note: cdrom mount command is slow
3764         }
3765         if ($res = __read_avail($sock, $timeout)) {
3766
3767             my @lines = split("\r?\n", $res);
3768
3769             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3770
3771             $res = join("\n", @lines);
3772             $res .= "\n";
3773         }
3774     };
3775
3776     my $err = $@;
3777
3778     if ($err) {
3779         syslog("err", "VM $vmid monitor command failed - $err");
3780         die $err;
3781     }
3782
3783     return $res;
3784 }
3785
3786 sub qemu_block_resize {
3787     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3788
3789     my $running = check_running($vmid);
3790
3791     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3792
3793     return if !$running;
3794
3795     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3796
3797 }
3798
3799 sub qemu_volume_snapshot {
3800     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3801
3802     my $running = check_running($vmid);
3803
3804     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3805         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3806     } else {
3807         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3808     }
3809 }
3810
3811 sub qemu_volume_snapshot_delete {
3812     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3813
3814     my $running = check_running($vmid);
3815
3816     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3817
3818     return if !$running;
3819
3820     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3821 }
3822
3823 sub set_migration_caps {
3824     my ($vmid) = @_;
3825
3826     my $cap_ref = [];
3827
3828     my $enabled_cap = {
3829         "auto-converge" => 1,
3830         "xbzrle" => 0,
3831         "x-rdma-pin-all" => 0,
3832         "zero-blocks" => 0,
3833     };
3834
3835     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3836
3837     for my $supported_capability (@$supported_capabilities) {
3838         push @$cap_ref, {
3839             capability => $supported_capability->{capability},
3840             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3841         };
3842     }
3843
3844     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3845 }
3846
3847 my $fast_plug_option = {
3848     'lock' => 1,
3849     'name' => 1,
3850     'onboot' => 1,
3851     'shares' => 1,
3852     'startup' => 1,
3853 };
3854
3855 # hotplug changes in [PENDING]
3856 # $selection hash can be used to only apply specified options, for
3857 # example: { cores => 1 } (only apply changed 'cores')
3858 # $errors ref is used to return error messages
3859 sub vmconfig_hotplug_pending {
3860     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3861
3862     my $defaults = load_defaults();
3863
3864     # commit values which do not have any impact on running VM first
3865     # Note: those option cannot raise errors, we we do not care about
3866     # $selection and always apply them.
3867
3868     my $add_error = sub {
3869         my ($opt, $msg) = @_;
3870         $errors->{$opt} = "hotplug problem - $msg";
3871     };
3872
3873     my $changes = 0;
3874     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3875         if ($fast_plug_option->{$opt}) {
3876             $conf->{$opt} = $conf->{pending}->{$opt};
3877             delete $conf->{pending}->{$opt};
3878             $changes = 1;
3879         }
3880     }
3881
3882     if ($changes) {
3883         update_config_nolock($vmid, $conf, 1);
3884         $conf = load_config($vmid); # update/reload
3885     }
3886
3887     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3888
3889     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3890     foreach my $opt (@delete) {
3891         next if $selection && !$selection->{$opt};
3892         eval {
3893             if ($opt eq 'hotplug') {
3894                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3895             } elsif ($opt eq 'tablet') {
3896                 die "skip\n" if !$hotplug_features->{usb};
3897                 if ($defaults->{tablet}) {
3898                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3899                 } else {
3900                     vm_deviceunplug($vmid, $conf, $opt);
3901                 }
3902             } elsif ($opt eq 'vcpus') {
3903                 die "skip\n" if !$hotplug_features->{cpu};
3904                 qemu_cpu_hotplug($vmid, $conf, undef);
3905             } elsif ($opt eq 'balloon') {
3906                 # enable balloon device is not hotpluggable
3907                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3908             } elsif ($fast_plug_option->{$opt}) {
3909                 # do nothing
3910             } elsif ($opt =~ m/^net(\d+)$/) {
3911                 die "skip\n" if !$hotplug_features->{network};
3912                 vm_deviceunplug($vmid, $conf, $opt);
3913             } elsif (valid_drivename($opt)) {
3914                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3915                 vm_deviceunplug($vmid, $conf, $opt);
3916                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3917             } elsif ($opt =~ m/^memory$/) {
3918                 die "skip\n" if !$hotplug_features->{memory};
3919                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3920             } elsif ($opt eq 'cpuunits') {
3921                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3922             } else {
3923                 die "skip\n";
3924             }
3925         };
3926         if (my $err = $@) {
3927             &$add_error($opt, $err) if $err ne "skip\n";
3928         } else {
3929             # save new config if hotplug was successful
3930             delete $conf->{$opt};
3931             vmconfig_undelete_pending_option($conf, $opt);
3932             update_config_nolock($vmid, $conf, 1);
3933             $conf = load_config($vmid); # update/reload
3934         }
3935     }
3936
3937     foreach my $opt (keys %{$conf->{pending}}) {
3938         next if $selection && !$selection->{$opt};
3939         my $value = $conf->{pending}->{$opt};
3940         eval {
3941             if ($opt eq 'hotplug') {
3942                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3943             } elsif ($opt eq 'tablet') {
3944                 die "skip\n" if !$hotplug_features->{usb};
3945                 if ($value == 1) {
3946                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3947                 } elsif ($value == 0) {
3948                     vm_deviceunplug($vmid, $conf, $opt);
3949                 }
3950             } elsif ($opt eq 'vcpus') {
3951                 die "skip\n" if !$hotplug_features->{cpu};
3952                 qemu_cpu_hotplug($vmid, $conf, $value);
3953             } elsif ($opt eq 'balloon') {
3954                 # enable/disable balloning device is not hotpluggable
3955                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3956                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3957                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3958
3959                 # allow manual ballooning if shares is set to zero
3960                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3961                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3962                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3963                 }
3964             } elsif ($opt =~ m/^net(\d+)$/) {
3965                 # some changes can be done without hotplug
3966                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3967                                     $vmid, $opt, $value);
3968             } elsif (valid_drivename($opt)) {
3969                 # some changes can be done without hotplug
3970                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3971                                      $vmid, $opt, $value, 1);
3972             } elsif ($opt =~ m/^memory$/) { #dimms
3973                 die "skip\n" if !$hotplug_features->{memory};
3974                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3975             } elsif ($opt eq 'cpuunits') {
3976                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
3977             } else {
3978                 die "skip\n";  # skip non-hot-pluggable options
3979             }
3980         };
3981         if (my $err = $@) {
3982             &$add_error($opt, $err) if $err ne "skip\n";
3983         } else {
3984             # save new config if hotplug was successful
3985             $conf->{$opt} = $value;
3986             delete $conf->{pending}->{$opt};
3987             update_config_nolock($vmid, $conf, 1);
3988             $conf = load_config($vmid); # update/reload
3989         }
3990     }
3991 }
3992
3993 sub vmconfig_apply_pending {
3994     my ($vmid, $conf, $storecfg) = @_;
3995
3996     # cold plug
3997
3998     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3999     foreach my $opt (@delete) { # delete
4000         die "internal error" if $opt =~ m/^unused/;
4001         $conf = load_config($vmid); # update/reload
4002         if (!defined($conf->{$opt})) {
4003             vmconfig_undelete_pending_option($conf, $opt);
4004             update_config_nolock($vmid, $conf, 1);
4005         } elsif (valid_drivename($opt)) {
4006             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
4007             vmconfig_undelete_pending_option($conf, $opt);
4008             delete $conf->{$opt};
4009             update_config_nolock($vmid, $conf, 1);
4010         } else {
4011             vmconfig_undelete_pending_option($conf, $opt);
4012             delete $conf->{$opt};
4013             update_config_nolock($vmid, $conf, 1);
4014         }
4015     }
4016
4017     $conf = load_config($vmid); # update/reload
4018
4019     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4020         $conf = load_config($vmid); # update/reload
4021
4022         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4023             # skip if nothing changed
4024         } elsif (valid_drivename($opt)) {
4025             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4026                 if defined($conf->{$opt});
4027             $conf->{$opt} = $conf->{pending}->{$opt};
4028         } else {
4029             $conf->{$opt} = $conf->{pending}->{$opt};
4030         }
4031
4032         delete $conf->{pending}->{$opt};
4033         update_config_nolock($vmid, $conf, 1);
4034     }
4035 }
4036
4037 my $safe_num_ne = sub {
4038     my ($a, $b) = @_;
4039
4040     return 0 if !defined($a) && !defined($b);
4041     return 1 if !defined($a);
4042     return 1 if !defined($b);
4043
4044     return $a != $b;
4045 };
4046
4047 my $safe_string_ne = sub {
4048     my ($a, $b) = @_;
4049
4050     return 0 if !defined($a) && !defined($b);
4051     return 1 if !defined($a);
4052     return 1 if !defined($b);
4053
4054     return $a ne $b;
4055 };
4056
4057 sub vmconfig_update_net {
4058     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4059
4060     my $newnet = parse_net($value);
4061
4062     if ($conf->{$opt}) {
4063         my $oldnet = parse_net($conf->{$opt});
4064
4065         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4066             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4067             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4068             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4069
4070             # for non online change, we try to hot-unplug
4071             die "skip\n" if !$hotplug;
4072             vm_deviceunplug($vmid, $conf, $opt);
4073         } else {
4074
4075             die "internal error" if $opt !~ m/net(\d+)/;
4076             my $iface = "tap${vmid}i$1";
4077
4078             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4079                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4080             }
4081
4082             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4083                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4084                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4085                 PVE::Network::tap_unplug($iface);
4086                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4087             }
4088
4089             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4090                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4091             }
4092
4093             return 1;
4094         }
4095     }
4096
4097     if ($hotplug) {
4098         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4099     } else {
4100         die "skip\n";
4101     }
4102 }
4103
4104 sub vmconfig_update_disk {
4105     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4106
4107     # fixme: do we need force?
4108
4109     my $drive = parse_drive($opt, $value);
4110
4111     if ($conf->{$opt}) {
4112
4113         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4114
4115             my $media = $drive->{media} || 'disk';
4116             my $oldmedia = $old_drive->{media} || 'disk';
4117             die "unable to change media type\n" if $media ne $oldmedia;
4118
4119             if (!drive_is_cdrom($old_drive)) {
4120
4121                 if ($drive->{file} ne $old_drive->{file}) {
4122
4123                     die "skip\n" if !$hotplug;
4124
4125                     # unplug and register as unused
4126                     vm_deviceunplug($vmid, $conf, $opt);
4127                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4128
4129                 } else {
4130                     # update existing disk
4131
4132                     # skip non hotpluggable value
4133                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4134                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4135                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4136                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4137                         die "skip\n";
4138                     }
4139
4140                     # apply throttle
4141                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4142                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4143                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4144                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4145                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4146                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4147                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4148                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4149                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4150                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4151                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4152                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4153
4154                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4155                                                    ($drive->{mbps} || 0)*1024*1024,
4156                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4157                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4158                                                    $drive->{iops} || 0,
4159                                                    $drive->{iops_rd} || 0,
4160                                                    $drive->{iops_wr} || 0,
4161                                                    ($drive->{mbps_max} || 0)*1024*1024,
4162                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4163                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4164                                                    $drive->{iops_max} || 0,
4165                                                    $drive->{iops_rd_max} || 0,
4166                                                    $drive->{iops_wr_max} || 0);
4167
4168                     }
4169
4170                     return 1;
4171                 }
4172
4173             } else { # cdrom
4174
4175                 if ($drive->{file} eq 'none') {
4176                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4177                 } else {
4178                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4179                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4180                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4181                 }
4182
4183                 return 1;
4184             }
4185         }
4186     }
4187
4188     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4189     # hotplug new disks
4190     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4191 }
4192
4193 sub vm_start {
4194     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4195
4196     lock_config($vmid, sub {
4197         my $conf = load_config($vmid, $migratedfrom);
4198
4199         die "you can't start a vm if it's a template\n" if is_template($conf);
4200
4201         check_lock($conf) if !$skiplock;
4202
4203         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4204
4205         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4206             vmconfig_apply_pending($vmid, $conf, $storecfg);
4207             $conf = load_config($vmid); # update/reload
4208         }
4209
4210         my $defaults = load_defaults();
4211
4212         # set environment variable useful inside network script
4213         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4214
4215         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4216
4217         my $migrate_port = 0;
4218         my $migrate_uri;
4219         if ($statefile) {
4220             if ($statefile eq 'tcp') {
4221                 my $localip = "localhost";
4222                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4223                 my $nodename = PVE::INotify::nodename();
4224                 if ($datacenterconf->{migration_unsecure}) {
4225                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4226                 }
4227                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4228                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4229                 $migrate_uri = "tcp:[${localip}]:${migrate_port}";
4230                 push @$cmd, '-incoming', $migrate_uri;
4231                 push @$cmd, '-S';
4232             } else {
4233                 push @$cmd, '-loadstate', $statefile;
4234             }
4235         } elsif ($paused) {
4236             push @$cmd, '-S';
4237         }
4238
4239         # host pci devices
4240         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4241           my $d = parse_hostpci($conf->{"hostpci$i"});
4242           next if !$d;
4243           my $pcidevices = $d->{pciid};
4244           foreach my $pcidevice (@$pcidevices) {
4245                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4246
4247                 my $info = pci_device_info("0000:$pciid");
4248                 die "IOMMU not present\n" if !check_iommu_support();
4249                 die "no pci device info for device '$pciid'\n" if !$info;
4250
4251                 if ($d->{driver} && $d->{driver} eq "vfio") {
4252                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4253                 } else {
4254                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
4255                 }
4256
4257                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4258           }
4259         }
4260
4261         PVE::Storage::activate_volumes($storecfg, $vollist);
4262
4263         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4264                     umask => 0077); };
4265         my $err = $@;
4266         die "start failed: $err" if $err;
4267
4268         print "migration listens on $migrate_uri\n" if $migrate_uri;
4269
4270         if ($statefile && $statefile ne 'tcp')  {
4271             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4272             warn $@ if $@;
4273         }
4274
4275         if ($migratedfrom) {
4276
4277             eval {
4278                 set_migration_caps($vmid);
4279             };
4280             warn $@ if $@;
4281
4282             if ($spice_port) {
4283                 print "spice listens on port $spice_port\n";
4284                 if ($spice_ticket) {
4285                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4286                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4287                 }
4288             }
4289
4290         } else {
4291
4292             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4293                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4294                     if $conf->{balloon};
4295             }
4296
4297             foreach my $opt (keys %$conf) {
4298                 next if $opt !~  m/^net\d+$/;
4299                 my $nicconf = parse_net($conf->{$opt});
4300                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4301             }
4302         }
4303
4304         vm_mon_cmd_nocheck($vmid, 'qom-set',
4305                     path => "machine/peripheral/balloon0",
4306                     property => "guest-stats-polling-interval",
4307                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4308
4309     });
4310 }
4311
4312 sub vm_mon_cmd {
4313     my ($vmid, $execute, %params) = @_;
4314
4315     my $cmd = { execute => $execute, arguments => \%params };
4316     vm_qmp_command($vmid, $cmd);
4317 }
4318
4319 sub vm_mon_cmd_nocheck {
4320     my ($vmid, $execute, %params) = @_;
4321
4322     my $cmd = { execute => $execute, arguments => \%params };
4323     vm_qmp_command($vmid, $cmd, 1);
4324 }
4325
4326 sub vm_qmp_command {
4327     my ($vmid, $cmd, $nocheck) = @_;
4328
4329     my $res;
4330
4331     my $timeout;
4332     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4333         $timeout = $cmd->{arguments}->{timeout};
4334         delete $cmd->{arguments}->{timeout};
4335     }
4336
4337     eval {
4338         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4339         my $sname = qmp_socket($vmid);
4340         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4341             my $qmpclient = PVE::QMPClient->new();
4342
4343             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4344         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4345             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4346                 if scalar(%{$cmd->{arguments}});
4347             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4348         } else {
4349             die "unable to open monitor socket\n";
4350         }
4351     };
4352     if (my $err = $@) {
4353         syslog("err", "VM $vmid qmp command failed - $err");
4354         die $err;
4355     }
4356
4357     return $res;
4358 }
4359
4360 sub vm_human_monitor_command {
4361     my ($vmid, $cmdline) = @_;
4362
4363     my $res;
4364
4365     my $cmd = {
4366         execute => 'human-monitor-command',
4367         arguments => { 'command-line' => $cmdline},
4368     };
4369
4370     return vm_qmp_command($vmid, $cmd);
4371 }
4372
4373 sub vm_commandline {
4374     my ($storecfg, $vmid) = @_;
4375
4376     my $conf = load_config($vmid);
4377
4378     my $defaults = load_defaults();
4379
4380     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4381
4382     return join(' ', @$cmd);
4383 }
4384
4385 sub vm_reset {
4386     my ($vmid, $skiplock) = @_;
4387
4388     lock_config($vmid, sub {
4389
4390         my $conf = load_config($vmid);
4391
4392         check_lock($conf) if !$skiplock;
4393
4394         vm_mon_cmd($vmid, "system_reset");
4395     });
4396 }
4397
4398 sub get_vm_volumes {
4399     my ($conf) = @_;
4400
4401     my $vollist = [];
4402     foreach_volid($conf, sub {
4403         my ($volid, $is_cdrom) = @_;
4404
4405         return if $volid =~ m|^/|;
4406
4407         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4408         return if !$sid;
4409
4410         push @$vollist, $volid;
4411     });
4412
4413     return $vollist;
4414 }
4415
4416 sub vm_stop_cleanup {
4417     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4418
4419     eval {
4420         fairsched_rmnod($vmid); # try to destroy group
4421
4422         if (!$keepActive) {
4423             my $vollist = get_vm_volumes($conf);
4424             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4425         }
4426
4427         foreach my $ext (qw(mon qmp pid vnc qga)) {
4428             unlink "/var/run/qemu-server/${vmid}.$ext";
4429         }
4430
4431         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4432     };
4433     warn $@ if $@; # avoid errors - just warn
4434 }
4435
4436 # Note: use $nockeck to skip tests if VM configuration file exists.
4437 # We need that when migration VMs to other nodes (files already moved)
4438 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4439 sub vm_stop {
4440     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4441
4442     $force = 1 if !defined($force) && !$shutdown;
4443
4444     if ($migratedfrom){
4445         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4446         kill 15, $pid if $pid;
4447         my $conf = load_config($vmid, $migratedfrom);
4448         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4449         return;
4450     }
4451
4452     lock_config($vmid, sub {
4453
4454         my $pid = check_running($vmid, $nocheck);
4455         return if !$pid;
4456
4457         my $conf;
4458         if (!$nocheck) {
4459             $conf = load_config($vmid);
4460             check_lock($conf) if !$skiplock;
4461             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4462                 my $opts = parse_startup($conf->{startup});
4463                 $timeout = $opts->{down} if $opts->{down};
4464             }
4465         }
4466
4467         $timeout = 60 if !defined($timeout);
4468
4469         eval {
4470             if ($shutdown) {
4471                 if (defined($conf) && $conf->{agent}) {
4472                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4473                 } else {
4474                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4475                 }
4476             } else {
4477                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4478             }
4479         };
4480         my $err = $@;
4481
4482         if (!$err) {
4483             my $count = 0;
4484             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4485                 $count++;
4486                 sleep 1;
4487             }
4488
4489             if ($count >= $timeout) {
4490                 if ($force) {
4491                     warn "VM still running - terminating now with SIGTERM\n";
4492                     kill 15, $pid;
4493                 } else {
4494                     die "VM quit/powerdown failed - got timeout\n";
4495                 }
4496             } else {
4497                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4498                 return;
4499             }
4500         } else {
4501             if ($force) {
4502                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4503                 kill 15, $pid;
4504             } else {
4505                 die "VM quit/powerdown failed\n";
4506             }
4507         }
4508
4509         # wait again
4510         $timeout = 10;
4511
4512         my $count = 0;
4513         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4514             $count++;
4515             sleep 1;
4516         }
4517
4518         if ($count >= $timeout) {
4519             warn "VM still running - terminating now with SIGKILL\n";
4520             kill 9, $pid;
4521             sleep 1;
4522         }
4523
4524         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4525    });
4526 }
4527
4528 sub vm_suspend {
4529     my ($vmid, $skiplock) = @_;
4530
4531     lock_config($vmid, sub {
4532
4533         my $conf = load_config($vmid);
4534
4535         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4536
4537         vm_mon_cmd($vmid, "stop");
4538     });
4539 }
4540
4541 sub vm_resume {
4542     my ($vmid, $skiplock) = @_;
4543
4544     lock_config($vmid, sub {
4545
4546         my $conf = load_config($vmid);
4547
4548         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4549
4550         vm_mon_cmd($vmid, "cont");
4551     });
4552 }
4553
4554 sub vm_sendkey {
4555     my ($vmid, $skiplock, $key) = @_;
4556
4557     lock_config($vmid, sub {
4558
4559         my $conf = load_config($vmid);
4560
4561         # there is no qmp command, so we use the human monitor command
4562         vm_human_monitor_command($vmid, "sendkey $key");
4563     });
4564 }
4565
4566 sub vm_destroy {
4567     my ($storecfg, $vmid, $skiplock) = @_;
4568
4569     lock_config($vmid, sub {
4570
4571         my $conf = load_config($vmid);
4572
4573         check_lock($conf) if !$skiplock;
4574
4575         if (!check_running($vmid)) {
4576             fairsched_rmnod($vmid); # try to destroy group
4577             destroy_vm($storecfg, $vmid);
4578         } else {
4579             die "VM $vmid is running - destroy failed\n";
4580         }
4581     });
4582 }
4583
4584 # pci helpers
4585
4586 sub file_write {
4587     my ($filename, $buf) = @_;
4588
4589     my $fh = IO::File->new($filename, "w");
4590     return undef if !$fh;
4591
4592     my $res = print $fh $buf;
4593
4594     $fh->close();
4595
4596     return $res;
4597 }
4598
4599 sub pci_device_info {
4600     my ($name) = @_;
4601
4602     my $res;
4603
4604     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4605     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4606
4607     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4608     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4609
4610     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4611     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4612
4613     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4614     return undef if !defined($product) || $product !~ s/^0x//;
4615
4616     $res = {
4617         name => $name,
4618         vendor => $vendor,
4619         product => $product,
4620         domain => $domain,
4621         bus => $bus,
4622         slot => $slot,
4623         func => $func,
4624         irq => $irq,
4625         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4626     };
4627
4628     return $res;
4629 }
4630
4631 sub pci_dev_reset {
4632     my ($dev) = @_;
4633
4634     my $name = $dev->{name};
4635
4636     my $fn = "$pcisysfs/devices/$name/reset";
4637
4638     return file_write($fn, "1");
4639 }
4640
4641 sub pci_dev_bind_to_stub {
4642     my ($dev) = @_;
4643
4644     my $name = $dev->{name};
4645
4646     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
4647     return 1 if -d $testdir;
4648
4649     my $data = "$dev->{vendor} $dev->{product}";
4650     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
4651
4652     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4653     if (!file_write($fn, $name)) {
4654         return undef if -f $fn;
4655     }
4656
4657     $fn = "$pcisysfs/drivers/pci-stub/bind";
4658     if (! -d $testdir) {
4659         return undef if !file_write($fn, $name);
4660     }
4661
4662     return -d $testdir;
4663 }
4664
4665 sub pci_dev_bind_to_vfio {
4666     my ($dev) = @_;
4667
4668     my $name = $dev->{name};
4669
4670     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4671
4672     if (!-d $vfio_basedir) {
4673         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4674     }
4675     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4676
4677     my $testdir = "$vfio_basedir/$name";
4678     return 1 if -d $testdir;
4679
4680     my $data = "$dev->{vendor} $dev->{product}";
4681     return undef if !file_write("$vfio_basedir/new_id", $data);
4682
4683     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4684     if (!file_write($fn, $name)) {
4685         return undef if -f $fn;
4686     }
4687
4688     $fn = "$vfio_basedir/bind";
4689     if (! -d $testdir) {
4690         return undef if !file_write($fn, $name);
4691     }
4692
4693     return -d $testdir;
4694 }
4695
4696 sub pci_dev_group_bind_to_vfio {
4697     my ($pciid) = @_;
4698
4699     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4700
4701     if (!-d $vfio_basedir) {
4702         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4703     }
4704     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4705
4706     # get IOMMU group devices
4707     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4708       my @devs = grep /^0000:/, readdir($D);
4709     closedir($D);
4710
4711     foreach my $pciid (@devs) {
4712         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4713
4714         # pci bridges, switches or root ports are not supported
4715         # they have a pci_bus subdirectory so skip them
4716         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4717
4718         my $info = pci_device_info($1);
4719         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4720     }
4721
4722     return 1;
4723 }
4724
4725 sub print_pci_addr {
4726     my ($id, $bridges) = @_;
4727
4728     my $res = '';
4729     my $devices = {
4730         piix3 => { bus => 0, addr => 1 },
4731         #addr2 : first videocard
4732         balloon0 => { bus => 0, addr => 3 },
4733         watchdog => { bus => 0, addr => 4 },
4734         scsihw0 => { bus => 0, addr => 5 },
4735         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4736         scsihw1 => { bus => 0, addr => 6 },
4737         ahci0 => { bus => 0, addr => 7 },
4738         qga0 => { bus => 0, addr => 8 },
4739         spice => { bus => 0, addr => 9 },
4740         virtio0 => { bus => 0, addr => 10 },
4741         virtio1 => { bus => 0, addr => 11 },
4742         virtio2 => { bus => 0, addr => 12 },
4743         virtio3 => { bus => 0, addr => 13 },
4744         virtio4 => { bus => 0, addr => 14 },
4745         virtio5 => { bus => 0, addr => 15 },
4746         hostpci0 => { bus => 0, addr => 16 },
4747         hostpci1 => { bus => 0, addr => 17 },
4748         net0 => { bus => 0, addr => 18 },
4749         net1 => { bus => 0, addr => 19 },
4750         net2 => { bus => 0, addr => 20 },
4751         net3 => { bus => 0, addr => 21 },
4752         net4 => { bus => 0, addr => 22 },
4753         net5 => { bus => 0, addr => 23 },
4754         vga1 => { bus => 0, addr => 24 },
4755         vga2 => { bus => 0, addr => 25 },
4756         vga3 => { bus => 0, addr => 26 },
4757         hostpci2 => { bus => 0, addr => 27 },
4758         hostpci3 => { bus => 0, addr => 28 },
4759         #addr29 : usb-host (pve-usb.cfg)
4760         'pci.1' => { bus => 0, addr => 30 },
4761         'pci.2' => { bus => 0, addr => 31 },
4762         'net6' => { bus => 1, addr => 1 },
4763         'net7' => { bus => 1, addr => 2 },
4764         'net8' => { bus => 1, addr => 3 },
4765         'net9' => { bus => 1, addr => 4 },
4766         'net10' => { bus => 1, addr => 5 },
4767         'net11' => { bus => 1, addr => 6 },
4768         'net12' => { bus => 1, addr => 7 },
4769         'net13' => { bus => 1, addr => 8 },
4770         'net14' => { bus => 1, addr => 9 },
4771         'net15' => { bus => 1, addr => 10 },
4772         'net16' => { bus => 1, addr => 11 },
4773         'net17' => { bus => 1, addr => 12 },
4774         'net18' => { bus => 1, addr => 13 },
4775         'net19' => { bus => 1, addr => 14 },
4776         'net20' => { bus => 1, addr => 15 },
4777         'net21' => { bus => 1, addr => 16 },
4778         'net22' => { bus => 1, addr => 17 },
4779         'net23' => { bus => 1, addr => 18 },
4780         'net24' => { bus => 1, addr => 19 },
4781         'net25' => { bus => 1, addr => 20 },
4782         'net26' => { bus => 1, addr => 21 },
4783         'net27' => { bus => 1, addr => 22 },
4784         'net28' => { bus => 1, addr => 23 },
4785         'net29' => { bus => 1, addr => 24 },
4786         'net30' => { bus => 1, addr => 25 },
4787         'net31' => { bus => 1, addr => 26 },
4788         'virtio6' => { bus => 2, addr => 1 },
4789         'virtio7' => { bus => 2, addr => 2 },
4790         'virtio8' => { bus => 2, addr => 3 },
4791         'virtio9' => { bus => 2, addr => 4 },
4792         'virtio10' => { bus => 2, addr => 5 },
4793         'virtio11' => { bus => 2, addr => 6 },
4794         'virtio12' => { bus => 2, addr => 7 },
4795         'virtio13' => { bus => 2, addr => 8 },
4796         'virtio14' => { bus => 2, addr => 9 },
4797         'virtio15' => { bus => 2, addr => 10 },
4798         'virtioscsi0' => { bus => 3, addr => 1 },
4799         'virtioscsi1' => { bus => 3, addr => 2 },
4800         'virtioscsi2' => { bus => 3, addr => 3 },
4801         'virtioscsi3' => { bus => 3, addr => 4 },
4802         'virtioscsi4' => { bus => 3, addr => 5 },
4803         'virtioscsi5' => { bus => 3, addr => 6 },
4804         'virtioscsi6' => { bus => 3, addr => 7 },
4805         'virtioscsi7' => { bus => 3, addr => 8 },
4806         'virtioscsi8' => { bus => 3, addr => 9 },
4807         'virtioscsi9' => { bus => 3, addr => 10 },
4808         'virtioscsi10' => { bus => 3, addr => 11 },
4809         'virtioscsi11' => { bus => 3, addr => 12 },
4810         'virtioscsi12' => { bus => 3, addr => 13 },
4811         'virtioscsi13' => { bus => 3, addr => 14 },
4812         'virtioscsi14' => { bus => 3, addr => 15 },
4813         'virtioscsi15' => { bus => 3, addr => 16 },
4814         'virtioscsi16' => { bus => 3, addr => 17 },
4815         'virtioscsi17' => { bus => 3, addr => 18 },
4816         'virtioscsi18' => { bus => 3, addr => 19 },
4817         'virtioscsi19' => { bus => 3, addr => 20 },
4818         'virtioscsi20' => { bus => 3, addr => 21 },
4819         'virtioscsi21' => { bus => 3, addr => 22 },
4820         'virtioscsi22' => { bus => 3, addr => 23 },
4821         'virtioscsi23' => { bus => 3, addr => 24 },
4822         'virtioscsi24' => { bus => 3, addr => 25 },
4823         'virtioscsi25' => { bus => 3, addr => 26 },
4824         'virtioscsi26' => { bus => 3, addr => 27 },
4825         'virtioscsi27' => { bus => 3, addr => 28 },
4826         'virtioscsi28' => { bus => 3, addr => 29 },
4827         'virtioscsi29' => { bus => 3, addr => 30 },
4828         'virtioscsi30' => { bus => 3, addr => 31 },
4829
4830     };
4831
4832     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4833            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4834            my $bus = $devices->{$id}->{bus};
4835            $res = ",bus=pci.$bus,addr=$addr";
4836            $bridges->{$bus} = 1 if $bridges;
4837     }
4838     return $res;
4839
4840 }
4841
4842 sub print_pcie_addr {
4843     my ($id) = @_;
4844
4845     my $res = '';
4846     my $devices = {
4847         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4848         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4849         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4850         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4851     };
4852
4853     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4854            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4855            my $bus = $devices->{$id}->{bus};
4856            $res = ",bus=$bus,addr=$addr";
4857     }
4858     return $res;
4859
4860 }
4861
4862 # vzdump restore implementaion
4863
4864 sub tar_archive_read_firstfile {
4865     my $archive = shift;
4866
4867     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4868
4869     # try to detect archive type first
4870     my $pid = open (TMP, "tar tf '$archive'|") ||
4871         die "unable to open file '$archive'\n";
4872     my $firstfile = <TMP>;
4873     kill 15, $pid;
4874     close TMP;
4875
4876     die "ERROR: archive contaions no data\n" if !$firstfile;
4877     chomp $firstfile;
4878
4879     return $firstfile;
4880 }
4881
4882 sub tar_restore_cleanup {
4883     my ($storecfg, $statfile) = @_;
4884
4885     print STDERR "starting cleanup\n";
4886
4887     if (my $fd = IO::File->new($statfile, "r")) {
4888         while (defined(my $line = <$fd>)) {
4889             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4890                 my $volid = $2;
4891                 eval {
4892                     if ($volid =~ m|^/|) {
4893                         unlink $volid || die 'unlink failed\n';
4894                     } else {
4895                         PVE::Storage::vdisk_free($storecfg, $volid);
4896                     }
4897                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4898                 };
4899                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4900             } else {
4901                 print STDERR "unable to parse line in statfile - $line";
4902             }
4903         }
4904         $fd->close();
4905     }
4906 }
4907
4908 sub restore_archive {
4909     my ($archive, $vmid, $user, $opts) = @_;
4910
4911     my $format = $opts->{format};
4912     my $comp;
4913
4914     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4915         $format = 'tar' if !$format;
4916         $comp = 'gzip';
4917     } elsif ($archive =~ m/\.tar$/) {
4918         $format = 'tar' if !$format;
4919     } elsif ($archive =~ m/.tar.lzo$/) {
4920         $format = 'tar' if !$format;
4921         $comp = 'lzop';
4922     } elsif ($archive =~ m/\.vma$/) {
4923         $format = 'vma' if !$format;
4924     } elsif ($archive =~ m/\.vma\.gz$/) {
4925         $format = 'vma' if !$format;
4926         $comp = 'gzip';
4927     } elsif ($archive =~ m/\.vma\.lzo$/) {
4928         $format = 'vma' if !$format;
4929         $comp = 'lzop';
4930     } else {
4931         $format = 'vma' if !$format; # default
4932     }
4933
4934     # try to detect archive format
4935     if ($format eq 'tar') {
4936         return restore_tar_archive($archive, $vmid, $user, $opts);
4937     } else {
4938         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4939     }
4940 }
4941
4942 sub restore_update_config_line {
4943     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4944
4945     return if $line =~ m/^\#qmdump\#/;
4946     return if $line =~ m/^\#vzdump\#/;
4947     return if $line =~ m/^lock:/;
4948     return if $line =~ m/^unused\d+:/;
4949     return if $line =~ m/^parent:/;
4950     return if $line =~ m/^template:/; # restored VM is never a template
4951
4952     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4953         # try to convert old 1.X settings
4954         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4955         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4956             my ($model, $macaddr) = split(/\=/, $devconfig);
4957             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4958             my $net = {
4959                 model => $model,
4960                 bridge => "vmbr$ind",
4961                 macaddr => $macaddr,
4962             };
4963             my $netstr = print_net($net);
4964
4965             print $outfd "net$cookie->{netcount}: $netstr\n";
4966             $cookie->{netcount}++;
4967         }
4968     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4969         my ($id, $netstr) = ($1, $2);
4970         my $net = parse_net($netstr);
4971         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4972         $netstr = print_net($net);
4973         print $outfd "$id: $netstr\n";
4974     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4975         my $virtdev = $1;
4976         my $value = $3;
4977         if ($line =~ m/backup=no/) {
4978             print $outfd "#$line";
4979         } elsif ($virtdev && $map->{$virtdev}) {
4980             my $di = parse_drive($virtdev, $value);
4981             delete $di->{format}; # format can change on restore
4982             $di->{file} = $map->{$virtdev};
4983             $value = print_drive($vmid, $di);
4984             print $outfd "$virtdev: $value\n";
4985         } else {
4986             print $outfd $line;
4987         }
4988     } else {
4989         print $outfd $line;
4990     }
4991 }
4992
4993 sub scan_volids {
4994     my ($cfg, $vmid) = @_;
4995
4996     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4997
4998     my $volid_hash = {};
4999     foreach my $storeid (keys %$info) {
5000         foreach my $item (@{$info->{$storeid}}) {
5001             next if !($item->{volid} && $item->{size});
5002             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5003             $volid_hash->{$item->{volid}} = $item;
5004         }
5005     }
5006
5007     return $volid_hash;
5008 }
5009
5010 sub get_used_paths {
5011     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5012
5013     my $used_path = {};
5014
5015     my $scan_config = sub {
5016         my ($cref, $snapname) = @_;
5017
5018         foreach my $key (keys %$cref) {
5019             my $value = $cref->{$key};
5020             if (valid_drivename($key)) {
5021                 next if $skip_drive && $key eq $skip_drive;
5022                 my $drive = parse_drive($key, $value);
5023                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5024                 if ($drive->{file} =~ m!^/!) {
5025                     $used_path->{$drive->{file}}++; # = 1;
5026                 } else {
5027                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5028                     next if !$storeid;
5029                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5030                     next if !$scfg;
5031                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5032                     $used_path->{$path}++; # = 1;
5033                 }
5034             }
5035         }
5036     };
5037
5038     &$scan_config($conf);
5039
5040     undef $skip_drive;
5041
5042     if ($scan_snapshots) {
5043         foreach my $snapname (keys %{$conf->{snapshots}}) {
5044             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5045         }
5046     }
5047
5048     return $used_path;
5049 }
5050
5051 sub update_disksize {
5052     my ($vmid, $conf, $volid_hash) = @_;
5053
5054     my $changes;
5055
5056     my $used = {};
5057
5058     # Note: it is allowed to define multiple storages with same path (alias), so
5059     # we need to check both 'volid' and real 'path' (two different volid can point
5060     # to the same path).
5061
5062     my $usedpath = {};
5063
5064     # update size info
5065     foreach my $opt (keys %$conf) {
5066         if (valid_drivename($opt)) {
5067             my $drive = parse_drive($opt, $conf->{$opt});
5068             my $volid = $drive->{file};
5069             next if !$volid;
5070
5071             $used->{$volid} = 1;
5072             if ($volid_hash->{$volid} &&
5073                 (my $path = $volid_hash->{$volid}->{path})) {
5074                 $usedpath->{$path} = 1;
5075             }
5076
5077             next if drive_is_cdrom($drive);
5078             next if !$volid_hash->{$volid};
5079
5080             $drive->{size} = $volid_hash->{$volid}->{size};
5081             my $new = print_drive($vmid, $drive);
5082             if ($new ne $conf->{$opt}) {
5083                 $changes = 1;
5084                 $conf->{$opt} = $new;
5085             }
5086         }
5087     }
5088
5089     # remove 'unusedX' entry if volume is used
5090     foreach my $opt (keys %$conf) {
5091         next if $opt !~ m/^unused\d+$/;
5092         my $volid = $conf->{$opt};
5093         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5094         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5095             $changes = 1;
5096             delete $conf->{$opt};
5097         }
5098     }
5099
5100     foreach my $volid (sort keys %$volid_hash) {
5101         next if $volid =~ m/vm-$vmid-state-/;
5102         next if $used->{$volid};
5103         my $path = $volid_hash->{$volid}->{path};
5104         next if !$path; # just to be sure
5105         next if $usedpath->{$path};
5106         $changes = 1;
5107         add_unused_volume($conf, $volid);
5108         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5109     }
5110
5111     return $changes;
5112 }
5113
5114 sub rescan {
5115     my ($vmid, $nolock) = @_;
5116
5117     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5118
5119     my $volid_hash = scan_volids($cfg, $vmid);
5120
5121     my $updatefn =  sub {
5122         my ($vmid) = @_;
5123
5124         my $conf = load_config($vmid);
5125
5126         check_lock($conf);
5127
5128         my $vm_volids = {};
5129         foreach my $volid (keys %$volid_hash) {
5130             my $info = $volid_hash->{$volid};
5131             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5132         }
5133
5134         my $changes = update_disksize($vmid, $conf, $vm_volids);
5135
5136         update_config_nolock($vmid, $conf, 1) if $changes;
5137     };
5138
5139     if (defined($vmid)) {
5140         if ($nolock) {
5141             &$updatefn($vmid);
5142         } else {
5143             lock_config($vmid, $updatefn, $vmid);
5144         }
5145     } else {
5146         my $vmlist = config_list();
5147         foreach my $vmid (keys %$vmlist) {
5148             if ($nolock) {
5149                 &$updatefn($vmid);
5150             } else {
5151                 lock_config($vmid, $updatefn, $vmid);
5152             }
5153         }
5154     }
5155 }
5156
5157 sub restore_vma_archive {
5158     my ($archive, $vmid, $user, $opts, $comp) = @_;
5159
5160     my $input = $archive eq '-' ? "<&STDIN" : undef;
5161     my $readfrom = $archive;
5162
5163     my $uncomp = '';
5164     if ($comp) {
5165         $readfrom = '-';
5166         my $qarchive = PVE::Tools::shellquote($archive);
5167         if ($comp eq 'gzip') {
5168             $uncomp = "zcat $qarchive|";
5169         } elsif ($comp eq 'lzop') {
5170             $uncomp = "lzop -d -c $qarchive|";
5171         } else {
5172             die "unknown compression method '$comp'\n";
5173         }
5174
5175     }
5176
5177     my $tmpdir = "/var/tmp/vzdumptmp$$";
5178     rmtree $tmpdir;
5179
5180     # disable interrupts (always do cleanups)
5181     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5182         warn "got interrupt - ignored\n";
5183     };
5184
5185     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5186     POSIX::mkfifo($mapfifo, 0600);
5187     my $fifofh;
5188
5189     my $openfifo = sub {
5190         open($fifofh, '>', $mapfifo) || die $!;
5191     };
5192
5193     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5194
5195     my $oldtimeout;
5196     my $timeout = 5;
5197
5198     my $devinfo = {};
5199
5200     my $rpcenv = PVE::RPCEnvironment::get();
5201
5202     my $conffile = config_file($vmid);
5203     my $tmpfn = "$conffile.$$.tmp";
5204
5205     # Note: $oldconf is undef if VM does not exists
5206     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5207
5208     my $print_devmap = sub {
5209         my $virtdev_hash = {};
5210
5211         my $cfgfn = "$tmpdir/qemu-server.conf";
5212
5213         # we can read the config - that is already extracted
5214         my $fh = IO::File->new($cfgfn, "r") ||
5215             "unable to read qemu-server.conf - $!\n";
5216
5217         while (defined(my $line = <$fh>)) {
5218             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5219                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5220                 die "archive does not contain data for drive '$virtdev'\n"
5221                     if !$devinfo->{$devname};
5222                 if (defined($opts->{storage})) {
5223                     $storeid = $opts->{storage} || 'local';
5224                 } elsif (!$storeid) {
5225                     $storeid = 'local';
5226                 }
5227                 $format = 'raw' if !$format;
5228                 $devinfo->{$devname}->{devname} = $devname;
5229                 $devinfo->{$devname}->{virtdev} = $virtdev;
5230                 $devinfo->{$devname}->{format} = $format;
5231                 $devinfo->{$devname}->{storeid} = $storeid;
5232
5233                 # check permission on storage
5234                 my $pool = $opts->{pool}; # todo: do we need that?
5235                 if ($user ne 'root@pam') {
5236                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5237                 }
5238
5239                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5240             }
5241         }
5242
5243         foreach my $devname (keys %$devinfo) {
5244             die "found no device mapping information for device '$devname'\n"
5245                 if !$devinfo->{$devname}->{virtdev};
5246         }
5247
5248         my $cfg = cfs_read_file('storage.cfg');
5249
5250         # create empty/temp config
5251         if ($oldconf) {
5252             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5253             foreach_drive($oldconf, sub {
5254                 my ($ds, $drive) = @_;
5255
5256                 return if drive_is_cdrom($drive);
5257
5258                 my $volid = $drive->{file};
5259
5260                 return if !$volid || $volid =~ m|^/|;
5261
5262                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5263                 return if !$path || !$owner || ($owner != $vmid);
5264
5265                 # Note: only delete disk we want to restore
5266                 # other volumes will become unused
5267                 if ($virtdev_hash->{$ds}) {
5268                     PVE::Storage::vdisk_free($cfg, $volid);
5269                 }
5270             });
5271         }
5272
5273         my $map = {};
5274         foreach my $virtdev (sort keys %$virtdev_hash) {
5275             my $d = $virtdev_hash->{$virtdev};
5276             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5277             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5278
5279             # test if requested format is supported
5280             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5281             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5282             $d->{format} = $defFormat if !$supported;
5283
5284             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5285                                                   $d->{format}, undef, $alloc_size);
5286             print STDERR "new volume ID is '$volid'\n";
5287             $d->{volid} = $volid;
5288             my $path = PVE::Storage::path($cfg, $volid);
5289
5290             my $write_zeros = 1;
5291             # fixme: what other storages types initialize volumes with zero?
5292             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5293                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5294                 $write_zeros = 0;
5295             }
5296
5297             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5298
5299             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5300             $map->{$virtdev} = $volid;
5301         }
5302
5303         $fh->seek(0, 0) || die "seek failed - $!\n";
5304
5305         my $outfd = new IO::File ($tmpfn, "w") ||
5306             die "unable to write config for VM $vmid\n";
5307
5308         my $cookie = { netcount => 0 };
5309         while (defined(my $line = <$fh>)) {
5310             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5311         }
5312
5313         $fh->close();
5314         $outfd->close();
5315     };
5316
5317     eval {
5318         # enable interrupts
5319         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5320             die "interrupted by signal\n";
5321         };
5322         local $SIG{ALRM} = sub { die "got timeout\n"; };
5323
5324         $oldtimeout = alarm($timeout);
5325
5326         my $parser = sub {
5327             my $line = shift;
5328
5329             print "$line\n";
5330
5331             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5332                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5333                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5334             } elsif ($line =~ m/^CTIME: /) {
5335                 # we correctly received the vma config, so we can disable
5336                 # the timeout now for disk allocation (set to 10 minutes, so
5337                 # that we always timeout if something goes wrong)
5338                 alarm(600);
5339                 &$print_devmap();
5340                 print $fifofh "done\n";
5341                 my $tmp = $oldtimeout || 0;
5342                 $oldtimeout = undef;
5343                 alarm($tmp);
5344                 close($fifofh);
5345             }
5346         };
5347
5348         print "restore vma archive: $cmd\n";
5349         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5350     };
5351     my $err = $@;
5352
5353     alarm($oldtimeout) if $oldtimeout;
5354
5355     unlink $mapfifo;
5356
5357     if ($err) {
5358         rmtree $tmpdir;
5359         unlink $tmpfn;
5360
5361         my $cfg = cfs_read_file('storage.cfg');
5362         foreach my $devname (keys %$devinfo) {
5363             my $volid = $devinfo->{$devname}->{volid};
5364             next if !$volid;
5365             eval {
5366                 if ($volid =~ m|^/|) {
5367                     unlink $volid || die 'unlink failed\n';
5368                 } else {
5369                     PVE::Storage::vdisk_free($cfg, $volid);
5370                 }
5371                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5372             };
5373             print STDERR "unable to cleanup '$volid' - $@" if $@;
5374         }
5375         die $err;
5376     }
5377
5378     rmtree $tmpdir;
5379
5380     rename($tmpfn, $conffile) ||
5381         die "unable to commit configuration file '$conffile'\n";
5382
5383     PVE::Cluster::cfs_update(); # make sure we read new file
5384
5385     eval { rescan($vmid, 1); };
5386     warn $@ if $@;
5387 }
5388
5389 sub restore_tar_archive {
5390     my ($archive, $vmid, $user, $opts) = @_;
5391
5392     if ($archive ne '-') {
5393         my $firstfile = tar_archive_read_firstfile($archive);
5394         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5395             if $firstfile ne 'qemu-server.conf';
5396     }
5397
5398     my $storecfg = cfs_read_file('storage.cfg');
5399
5400     # destroy existing data - keep empty config
5401     my $vmcfgfn = config_file($vmid);
5402     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5403
5404     my $tocmd = "/usr/lib/qemu-server/qmextract";
5405
5406     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5407     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5408     $tocmd .= ' --prealloc' if $opts->{prealloc};
5409     $tocmd .= ' --info' if $opts->{info};
5410
5411     # tar option "xf" does not autodetect compression when read from STDIN,
5412     # so we pipe to zcat
5413     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5414         PVE::Tools::shellquote("--to-command=$tocmd");
5415
5416     my $tmpdir = "/var/tmp/vzdumptmp$$";
5417     mkpath $tmpdir;
5418
5419     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5420     local $ENV{VZDUMP_VMID} = $vmid;
5421     local $ENV{VZDUMP_USER} = $user;
5422
5423     my $conffile = config_file($vmid);
5424     my $tmpfn = "$conffile.$$.tmp";
5425
5426     # disable interrupts (always do cleanups)
5427     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5428         print STDERR "got interrupt - ignored\n";
5429     };
5430
5431     eval {
5432         # enable interrupts
5433         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5434             die "interrupted by signal\n";
5435         };
5436
5437         if ($archive eq '-') {
5438             print "extracting archive from STDIN\n";
5439             run_command($cmd, input => "<&STDIN");
5440         } else {
5441             print "extracting archive '$archive'\n";
5442             run_command($cmd);
5443         }
5444
5445         return if $opts->{info};
5446
5447         # read new mapping
5448         my $map = {};
5449         my $statfile = "$tmpdir/qmrestore.stat";
5450         if (my $fd = IO::File->new($statfile, "r")) {
5451             while (defined (my $line = <$fd>)) {
5452                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5453                     $map->{$1} = $2 if $1;
5454                 } else {
5455                     print STDERR "unable to parse line in statfile - $line\n";
5456                 }
5457             }
5458             $fd->close();
5459         }
5460
5461         my $confsrc = "$tmpdir/qemu-server.conf";
5462
5463         my $srcfd = new IO::File($confsrc, "r") ||
5464             die "unable to open file '$confsrc'\n";
5465
5466         my $outfd = new IO::File ($tmpfn, "w") ||
5467             die "unable to write config for VM $vmid\n";
5468
5469         my $cookie = { netcount => 0 };
5470         while (defined (my $line = <$srcfd>)) {
5471             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5472         }
5473
5474         $srcfd->close();
5475         $outfd->close();
5476     };
5477     my $err = $@;
5478
5479     if ($err) {
5480
5481         unlink $tmpfn;
5482
5483         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5484
5485         die $err;
5486     }
5487
5488     rmtree $tmpdir;
5489
5490     rename $tmpfn, $conffile ||
5491         die "unable to commit configuration file '$conffile'\n";
5492
5493     PVE::Cluster::cfs_update(); # make sure we read new file
5494
5495     eval { rescan($vmid, 1); };
5496     warn $@ if $@;
5497 };
5498
5499
5500 # Internal snapshots
5501
5502 # NOTE: Snapshot create/delete involves several non-atomic
5503 # action, and can take a long time.
5504 # So we try to avoid locking the file and use 'lock' variable
5505 # inside the config file instead.
5506
5507 my $snapshot_copy_config = sub {
5508     my ($source, $dest) = @_;
5509
5510     foreach my $k (keys %$source) {
5511         next if $k eq 'snapshots';
5512         next if $k eq 'snapstate';
5513         next if $k eq 'snaptime';
5514         next if $k eq 'vmstate';
5515         next if $k eq 'lock';
5516         next if $k eq 'digest';
5517         next if $k eq 'description';
5518         next if $k =~ m/^unused\d+$/;
5519
5520         $dest->{$k} = $source->{$k};
5521     }
5522 };
5523
5524 my $snapshot_apply_config = sub {
5525     my ($conf, $snap) = @_;
5526
5527     # copy snapshot list
5528     my $newconf = {
5529         snapshots => $conf->{snapshots},
5530     };
5531
5532     # keep description and list of unused disks
5533     foreach my $k (keys %$conf) {
5534         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5535         $newconf->{$k} = $conf->{$k};
5536     }
5537
5538     &$snapshot_copy_config($snap, $newconf);
5539
5540     return $newconf;
5541 };
5542
5543 sub foreach_writable_storage {
5544     my ($conf, $func) = @_;
5545
5546     my $sidhash = {};
5547
5548     foreach my $ds (keys %$conf) {
5549         next if !valid_drivename($ds);
5550
5551         my $drive = parse_drive($ds, $conf->{$ds});
5552         next if !$drive;
5553         next if drive_is_cdrom($drive);
5554
5555         my $volid = $drive->{file};
5556
5557         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5558         $sidhash->{$sid} = $sid if $sid;
5559     }
5560
5561     foreach my $sid (sort keys %$sidhash) {
5562         &$func($sid);
5563     }
5564 }
5565
5566 my $alloc_vmstate_volid = sub {
5567     my ($storecfg, $vmid, $conf, $snapname) = @_;
5568
5569     # Note: we try to be smart when selecting a $target storage
5570
5571     my $target;
5572
5573     # search shared storage first
5574     foreach_writable_storage($conf, sub {
5575         my ($sid) = @_;
5576         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5577         return if !$scfg->{shared};
5578
5579         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5580     });
5581
5582     if (!$target) {
5583         # now search local storage
5584         foreach_writable_storage($conf, sub {
5585             my ($sid) = @_;
5586             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5587             return if $scfg->{shared};
5588
5589             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5590         });
5591     }
5592
5593     $target = 'local' if !$target;
5594
5595     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5596     # we abort live save after $conf->{memory}, so we need at max twice that space
5597     my $size = $conf->{memory}*2 + $driver_state_size;
5598
5599     my $name = "vm-$vmid-state-$snapname";
5600     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5601     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5602     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5603
5604     return $volid;
5605 };
5606
5607 my $snapshot_prepare = sub {
5608     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5609
5610     my $snap;
5611
5612     my $updatefn =  sub {
5613
5614         my $conf = load_config($vmid);
5615
5616         die "you can't take a snapshot if it's a template\n"
5617             if is_template($conf);
5618
5619         check_lock($conf);
5620
5621         $conf->{lock} = 'snapshot';
5622
5623         die "snapshot name '$snapname' already used\n"
5624             if defined($conf->{snapshots}->{$snapname});
5625
5626         my $storecfg = PVE::Storage::config();
5627         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5628
5629         $snap = $conf->{snapshots}->{$snapname} = {};
5630
5631         if ($save_vmstate && check_running($vmid)) {
5632             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5633         }
5634
5635         &$snapshot_copy_config($conf, $snap);
5636
5637         $snap->{snapstate} = "prepare";
5638         $snap->{snaptime} = time();
5639         $snap->{description} = $comment if $comment;
5640
5641         # always overwrite machine if we save vmstate. This makes sure we
5642         # can restore it later using correct machine type
5643         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5644
5645         update_config_nolock($vmid, $conf, 1);
5646     };
5647
5648     lock_config($vmid, $updatefn);
5649
5650     return $snap;
5651 };
5652
5653 my $snapshot_commit = sub {
5654     my ($vmid, $snapname) = @_;
5655
5656     my $updatefn = sub {
5657
5658         my $conf = load_config($vmid);
5659
5660         die "missing snapshot lock\n"
5661             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5662
5663         my $has_machine_config = defined($conf->{machine});
5664
5665         my $snap = $conf->{snapshots}->{$snapname};
5666
5667         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5668
5669         die "wrong snapshot state\n"
5670             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5671
5672         delete $snap->{snapstate};
5673         delete $conf->{lock};
5674
5675         my $newconf = &$snapshot_apply_config($conf, $snap);
5676
5677         delete $newconf->{machine} if !$has_machine_config;
5678
5679         $newconf->{parent} = $snapname;
5680
5681         update_config_nolock($vmid, $newconf, 1);
5682     };
5683
5684     lock_config($vmid, $updatefn);
5685 };
5686
5687 sub snapshot_rollback {
5688     my ($vmid, $snapname) = @_;
5689
5690     my $prepare = 1;
5691
5692     my $storecfg = PVE::Storage::config();
5693
5694     my $conf = load_config($vmid);
5695
5696     my $get_snapshot_config = sub {
5697
5698         die "you can't rollback if vm is a template\n" if is_template($conf);
5699
5700         my $res = $conf->{snapshots}->{$snapname};
5701
5702         die "snapshot '$snapname' does not exist\n" if !defined($res);
5703
5704         return $res;
5705     };
5706
5707     my $snap = &$get_snapshot_config();
5708
5709     foreach_drive($snap, sub {
5710         my ($ds, $drive) = @_;
5711
5712         return if drive_is_cdrom($drive);
5713
5714         my $volid = $drive->{file};
5715
5716         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5717     });
5718
5719     my $updatefn = sub {
5720
5721         $conf = load_config($vmid);
5722
5723         $snap = &$get_snapshot_config();
5724
5725         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5726             if $snap->{snapstate};
5727
5728         if ($prepare) {
5729             check_lock($conf);
5730             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5731         }
5732
5733         die "unable to rollback vm $vmid: vm is running\n"
5734             if check_running($vmid);
5735
5736         if ($prepare) {
5737             $conf->{lock} = 'rollback';
5738         } else {
5739             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5740             delete $conf->{lock};
5741         }
5742
5743         my $forcemachine;
5744
5745         if (!$prepare) {
5746             my $has_machine_config = defined($conf->{machine});
5747
5748             # copy snapshot config to current config
5749             $conf = &$snapshot_apply_config($conf, $snap);
5750             $conf->{parent} = $snapname;
5751
5752             # Note: old code did not store 'machine', so we try to be smart
5753             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5754             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5755             # we remove the 'machine' configuration if not explicitly specified
5756             # in the original config.
5757             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5758         }
5759
5760         update_config_nolock($vmid, $conf, 1);
5761
5762         if (!$prepare && $snap->{vmstate}) {
5763             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5764             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5765         }
5766     };
5767
5768     lock_config($vmid, $updatefn);
5769
5770     foreach_drive($snap, sub {
5771         my ($ds, $drive) = @_;
5772
5773         return if drive_is_cdrom($drive);
5774
5775         my $volid = $drive->{file};
5776         my $device = "drive-$ds";
5777
5778         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5779     });
5780
5781     $prepare = 0;
5782     lock_config($vmid, $updatefn);
5783 }
5784
5785 my $savevm_wait = sub {
5786     my ($vmid) = @_;
5787
5788     for(;;) {
5789         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5790         if (!$stat->{status}) {
5791             die "savevm not active\n";
5792         } elsif ($stat->{status} eq 'active') {
5793             sleep(1);
5794             next;
5795         } elsif ($stat->{status} eq 'completed') {
5796             last;
5797         } else {
5798             die "query-savevm returned status '$stat->{status}'\n";
5799         }
5800     }
5801 };
5802
5803 sub do_snapshots_with_qemu {
5804     my ($storecfg, $volid) = @_;
5805
5806     my $storage_name = PVE::Storage::parse_volume_id($volid);
5807
5808     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}} ){
5809         return 1;
5810     }
5811
5812     if ($volid =~ m/\.(qcow2|qed)$/){
5813         return 1;
5814     }
5815
5816     return undef;
5817 }
5818
5819 sub snapshot_create {
5820     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5821
5822     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5823
5824     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5825
5826     my $config = load_config($vmid);
5827
5828     my $running = check_running($vmid);
5829
5830     my $freezefs = $running && $config->{agent};
5831     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5832
5833     my $drivehash = {};
5834
5835     if ($freezefs) {
5836         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5837         warn "guest-fsfreeze-freeze problems - $@" if $@;
5838     }
5839
5840     eval {
5841         # create internal snapshots of all drives
5842
5843         my $storecfg = PVE::Storage::config();
5844
5845         if ($running) {
5846             if ($snap->{vmstate}) {
5847                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5848                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5849                 &$savevm_wait($vmid);
5850             } else {
5851                 vm_mon_cmd($vmid, "savevm-start");
5852             }
5853         };
5854
5855         foreach_drive($snap, sub {
5856             my ($ds, $drive) = @_;
5857
5858             return if drive_is_cdrom($drive);
5859
5860             my $volid = $drive->{file};
5861             my $device = "drive-$ds";
5862
5863             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5864             $drivehash->{$ds} = 1;
5865        });
5866     };
5867     my $err = $@;
5868
5869     if ($running) {
5870         eval { vm_mon_cmd($vmid, "savevm-end")  };
5871         warn $@ if $@;
5872
5873         if ($freezefs) {
5874             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5875             warn "guest-fsfreeze-thaw problems - $@" if $@;
5876         }
5877
5878         # savevm-end is async, we need to wait
5879         for (;;) {
5880             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5881             if (!$stat->{bytes}) {
5882                 last;
5883             } else {
5884                 print "savevm not yet finished\n";
5885                 sleep(1);
5886                 next;
5887             }
5888         }
5889     }
5890
5891     if ($err) {
5892         warn "snapshot create failed: starting cleanup\n";
5893         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5894         warn $@ if $@;
5895         die $err;
5896     }
5897
5898     &$snapshot_commit($vmid, $snapname);
5899 }
5900
5901 # Note: $drivehash is only set when called from snapshot_create.
5902 sub snapshot_delete {
5903     my ($vmid, $snapname, $force, $drivehash) = @_;
5904
5905     my $prepare = 1;
5906
5907     my $snap;
5908     my $unused = [];
5909
5910     my $unlink_parent = sub {
5911         my ($confref, $new_parent) = @_;
5912
5913         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5914             if ($new_parent) {
5915                 $confref->{parent} = $new_parent;
5916             } else {
5917                 delete $confref->{parent};
5918             }
5919         }
5920     };
5921
5922     my $updatefn =  sub {
5923         my ($remove_drive) = @_;
5924
5925         my $conf = load_config($vmid);
5926
5927         if (!$drivehash) {
5928             check_lock($conf);
5929             die "you can't delete a snapshot if vm is a template\n"
5930                 if is_template($conf);
5931         }
5932
5933         $snap = $conf->{snapshots}->{$snapname};
5934
5935         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5936
5937         # remove parent refs
5938         if (!$prepare) {
5939             &$unlink_parent($conf, $snap->{parent});
5940             foreach my $sn (keys %{$conf->{snapshots}}) {
5941                 next if $sn eq $snapname;
5942                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
5943             }
5944         }
5945
5946         if ($remove_drive) {
5947             if ($remove_drive eq 'vmstate') {
5948                 delete $snap->{$remove_drive};
5949             } else {
5950                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
5951                 my $volid = $drive->{file};
5952                 delete $snap->{$remove_drive};
5953                 add_unused_volume($conf, $volid);
5954             }
5955         }
5956
5957         if ($prepare) {
5958             $snap->{snapstate} = 'delete';
5959         } else {
5960             delete $conf->{snapshots}->{$snapname};
5961             delete $conf->{lock} if $drivehash;
5962             foreach my $volid (@$unused) {
5963                 add_unused_volume($conf, $volid);
5964             }
5965         }
5966
5967         update_config_nolock($vmid, $conf, 1);
5968     };
5969
5970     lock_config($vmid, $updatefn);
5971
5972     # now remove vmstate file
5973
5974     my $storecfg = PVE::Storage::config();
5975
5976     if ($snap->{vmstate}) {
5977         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5978         if (my $err = $@) {
5979             die $err if !$force;
5980             warn $err;
5981         }
5982         # save changes (remove vmstate from snapshot)
5983         lock_config($vmid, $updatefn, 'vmstate') if !$force;
5984     };
5985
5986     # now remove all internal snapshots
5987     foreach_drive($snap, sub {
5988         my ($ds, $drive) = @_;
5989
5990         return if drive_is_cdrom($drive);
5991
5992         my $volid = $drive->{file};
5993         my $device = "drive-$ds";
5994
5995         if (!$drivehash || $drivehash->{$ds}) {
5996             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
5997             if (my $err = $@) {
5998                 die $err if !$force;
5999                 warn $err;
6000             }
6001         }
6002
6003         # save changes (remove drive fron snapshot)
6004         lock_config($vmid, $updatefn, $ds) if !$force;
6005         push @$unused, $volid;
6006     });
6007
6008     # now cleanup config
6009     $prepare = 0;
6010     lock_config($vmid, $updatefn);
6011 }
6012
6013 sub has_feature {
6014     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6015
6016     my $err;
6017     foreach_drive($conf, sub {
6018         my ($ds, $drive) = @_;
6019
6020         return if drive_is_cdrom($drive);
6021         my $volid = $drive->{file};
6022         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6023     });
6024
6025     return $err ? 0 : 1;
6026 }
6027
6028 sub template_create {
6029     my ($vmid, $conf, $disk) = @_;
6030
6031     my $storecfg = PVE::Storage::config();
6032
6033     foreach_drive($conf, sub {
6034         my ($ds, $drive) = @_;
6035
6036         return if drive_is_cdrom($drive);
6037         return if $disk && $ds ne $disk;
6038
6039         my $volid = $drive->{file};
6040         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6041
6042         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6043         $drive->{file} = $voliddst;
6044         $conf->{$ds} = print_drive($vmid, $drive);
6045         update_config_nolock($vmid, $conf, 1);
6046     });
6047 }
6048
6049 sub is_template {
6050     my ($conf) = @_;
6051
6052     return 1 if defined $conf->{template} && $conf->{template} == 1;
6053 }
6054
6055 sub qemu_img_convert {
6056     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6057
6058     my $storecfg = PVE::Storage::config();
6059     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6060     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6061
6062     if ($src_storeid && $dst_storeid) {
6063         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6064         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6065
6066         my $src_format = qemu_img_format($src_scfg, $src_volname);
6067         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6068
6069         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6070         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6071
6072         my $cmd = [];
6073         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6074         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6075         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6076
6077         my $parser = sub {
6078             my $line = shift;
6079             if($line =~ m/\((\S+)\/100\%\)/){
6080                 my $percent = $1;
6081                 my $transferred = int($size * $percent / 100);
6082                 my $remaining = $size - $transferred;
6083
6084                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6085             }
6086
6087         };
6088
6089         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6090         my $err = $@;
6091         die "copy failed: $err" if $err;
6092     }
6093 }
6094
6095 sub qemu_img_format {
6096     my ($scfg, $volname) = @_;
6097
6098     if ($scfg->{path} && $volname =~ m/\.(raw|qcow2|qed|vmdk)$/) {
6099         return $1;
6100     } elsif ($scfg->{type} eq 'iscsi') {
6101         return "host_device";
6102     } else {
6103         return "raw";
6104     }
6105 }
6106
6107 sub qemu_drive_mirror {
6108     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6109
6110     my $storecfg = PVE::Storage::config();
6111     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6112
6113     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6114
6115     my $format;
6116     if ($dst_volname =~ m/\.(raw|qcow2)$/){
6117         $format = $1;
6118     }
6119
6120     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6121
6122     #drive-mirror is doing lseek on source image before starting, and this can take a lot of time for big nfs volume
6123     #during this time, qmp socket is hanging
6124     #http://lists.nongnu.org/archive/html/qemu-devel/2015-05/msg01838.html
6125     #so we need to setup a big timeout
6126     my $opts = { timeout => 14400, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6127     $opts->{format} = $format if $format;
6128
6129     print "drive mirror is starting : this step can take some minutes/hours, depend of disk size and storage speed\n";
6130
6131     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6132     eval {
6133         while (1) {
6134             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6135             my $stat = @$stats[0];
6136             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6137             die "error job is not mirroring" if $stat->{type} ne "mirror";
6138
6139             my $busy = $stat->{busy};
6140             my $ready = $stat->{ready};
6141
6142             if (my $total = $stat->{len}) {
6143                 my $transferred = $stat->{offset} || 0;
6144                 my $remaining = $total - $transferred;
6145                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6146
6147                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6148             }
6149
6150
6151             if ($stat->{ready} eq 'true') {
6152
6153                 last if $vmiddst != $vmid;
6154
6155                 # try to switch the disk if source and destination are on the same guest
6156                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6157                 last if !$@;
6158                 die $@ if $@ !~ m/cannot be completed/;
6159             }
6160             sleep 1;
6161         }
6162
6163
6164     };
6165     my $err = $@;
6166
6167     my $cancel_job = sub {
6168         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6169         while (1) {
6170             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6171             my $stat = @$stats[0];
6172             last if !$stat;
6173             sleep 1;
6174         }
6175     };
6176
6177     if ($err) {
6178         eval { &$cancel_job(); };
6179         die "mirroring error: $err";
6180     }
6181
6182     if ($vmiddst != $vmid) {
6183         # if we clone a disk for a new target vm, we don't switch the disk
6184         &$cancel_job(); # so we call block-job-cancel
6185     }
6186 }
6187
6188 sub clone_disk {
6189     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6190         $newvmid, $storage, $format, $full, $newvollist) = @_;
6191
6192     my $newvolid;
6193
6194     if (!$full) {
6195         print "create linked clone of drive $drivename ($drive->{file})\n";
6196         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6197         push @$newvollist, $newvolid;
6198     } else {
6199         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6200         $storeid = $storage if $storage;
6201
6202         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6203         if (!$format) {
6204             $format = $drive->{format} || $defFormat;
6205         }
6206
6207         # test if requested format is supported - else use default
6208         my $supported = grep { $_ eq $format } @$validFormats;
6209         $format = $defFormat if !$supported;
6210
6211         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6212
6213         print "create full clone of drive $drivename ($drive->{file})\n";
6214         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6215         push @$newvollist, $newvolid;
6216
6217         if (!$running || $snapname) {
6218             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6219         } else {
6220             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6221         }
6222     }
6223
6224     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6225
6226     my $disk = $drive;
6227     $disk->{format} = undef;
6228     $disk->{file} = $newvolid;
6229     $disk->{size} = $size;
6230
6231     return $disk;
6232 }
6233
6234 # this only works if VM is running
6235 sub get_current_qemu_machine {
6236     my ($vmid) = @_;
6237
6238     my $cmd = { execute => 'query-machines', arguments => {} };
6239     my $res = vm_qmp_command($vmid, $cmd);
6240
6241     my ($current, $default);
6242     foreach my $e (@$res) {
6243         $default = $e->{name} if $e->{'is-default'};
6244         $current = $e->{name} if $e->{'is-current'};
6245     }
6246
6247     # fallback to the default machine if current is not supported by qemu
6248     return $current || $default || 'pc';
6249 }
6250
6251 sub qemu_machine_feature_enabled {
6252     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6253
6254     my $current_major;
6255     my $current_minor;
6256
6257     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6258
6259         $current_major = $3;
6260         $current_minor = $4;
6261
6262     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6263
6264         $current_major = $1;
6265         $current_minor = $2;
6266     }
6267
6268     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6269
6270
6271 }
6272
6273 sub lspci {
6274
6275     my $devices = {};
6276
6277     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6278             my (undef, $id, $function) = @_;
6279             my $res = { id => $id, function => $function};
6280             push @{$devices->{$id}}, $res;
6281     });
6282
6283     return $devices;
6284 }
6285
6286 sub vm_iothreads_list {
6287     my ($vmid) = @_;
6288
6289     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6290
6291     my $iothreads = {};
6292     foreach my $iothread (@$res) {
6293         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6294     }
6295
6296     return $iothreads;
6297 }
6298
6299 sub scsihw_infos {
6300     my ($conf, $drive) = @_;
6301
6302     my $maxdev = 0;
6303
6304     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6305         $maxdev = 7;
6306     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6307         $maxdev = 1;
6308     } else {
6309         $maxdev = 256;
6310     }
6311
6312     my $controller = int($drive->{index} / $maxdev);
6313     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6314
6315     return ($maxdev, $controller, $controller_prefix);
6316 }
6317
6318 1;