PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  60     description => "Some command save/restore state from this location.",
  61     type => 'string',
  62     maxLength => 128,
  63     optional => 1,
  64 });
  65
  66 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  67     description => "The name of the snapshot.",
  68     type => 'string', format => 'pve-configid',
  69     maxLength => 40,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  73     type => 'string',
  74     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  75     description => "The drive's backing file's data format.",
  76     optional => 1,
  77 });
  78
  79 #no warnings 'redefine';
  80
  81 sub cgroups_write {
  82    my ($controller, $vmid, $option, $value) = @_;
  83
  84    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  85    PVE::ProcFSTools::write_proc_entry($path, $value);
  86
  87 }
  88
  89 my $nodename = PVE::INotify::nodename();
  90
  91 mkdir "/etc/pve/nodes/$nodename";
  92 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  93 mkdir $confdir;
  94
  95 my $var_run_tmpdir = "/var/run/qemu-server";
  96 mkdir $var_run_tmpdir;
  97
  98 my $lock_dir = "/var/lock/qemu-server";
  99 mkdir $lock_dir;
 100
 101 my $pcisysfs = "/sys/bus/pci";
 102
 103 my $cpu_vendor_list = {
 104     # Intel CPUs
 105     486 => 'GenuineIntel',
 106     pentium => 'GenuineIntel',
 107     pentium2  => 'GenuineIntel',
 108     pentium3  => 'GenuineIntel',
 109     coreduo => 'GenuineIntel',
 110     core2duo => 'GenuineIntel',
 111     Conroe  => 'GenuineIntel',
 112     Penryn  => 'GenuineIntel',
 113     Nehalem  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     SandyBridge => 'GenuineIntel',
 116     IvyBridge => 'GenuineIntel',
 117     Haswell => 'GenuineIntel',
 118     'Haswell-noTSX' => 'GenuineIntel',
 119     Broadwell => 'GenuineIntel',
 120     'Broadwell-noTSX' => 'GenuineIntel',
 121     'Skylake-Client' => 'GenuineIntel',
 122
 123     # AMD CPUs
 124     athlon => 'AuthenticAMD',
 125     phenom  => 'AuthenticAMD',
 126     Opteron_G1  => 'AuthenticAMD',
 127     Opteron_G2  => 'AuthenticAMD',
 128     Opteron_G3  => 'AuthenticAMD',
 129     Opteron_G4  => 'AuthenticAMD',
 130     Opteron_G5  => 'AuthenticAMD',
 131
 132     # generic types, use vendor from host node
 133     host => 'default',
 134     kvm32 => 'default',
 135     kvm64 => 'default',
 136     qemu32 => 'default',
 137     qemu64 => 'default',
 138 };
 139
 140 my $cpu_fmt = {
 141     cputype => {
 142         description => "Emulated CPU type.",
 143         type => 'string',
 144         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 145         default => 'kvm64',
 146         default_key => 1,
 147     },
 148     hidden => {
 149         description => "Do not identify as a KVM virtual machine.",
 150         type => 'boolean',
 151         optional => 1,
 152         default => 0
 153     },
 154 };
 155
 156 my $watchdog_fmt = {
 157     model => {
 158         default_key => 1,
 159         type => 'string',
 160         enum => [qw(i6300esb ib700)],
 161         description => "Watchdog type to emulate.",
 162         default => 'i6300esb',
 163         optional => 1,
 164     },
 165     action => {
 166         type => 'string',
 167         enum => [qw(reset shutdown poweroff pause debug none)],
 168         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 169         optional => 1,
 170     },
 171 };
 172 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 173
 174 my $confdesc = {
 175     onboot => {
 176         optional => 1,
 177         type => 'boolean',
 178         description => "Specifies whether a VM will be started during system bootup.",
 179         default => 0,
 180     },
 181     autostart => {
 182         optional => 1,
 183         type => 'boolean',
 184         description => "Automatic restart after crash (currently ignored).",
 185         default => 0,
 186     },
 187     hotplug => {
 188         optional => 1,
 189         type => 'string', format => 'pve-hotplug-features',
 190         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 191         default => 'network,disk,usb',
 192     },
 193     reboot => {
 194         optional => 1,
 195         type => 'boolean',
 196         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 197         default => 1,
 198     },
 199     lock => {
 200         optional => 1,
 201         type => 'string',
 202         description => "Lock/unlock the VM.",
 203         enum => [qw(migrate backup snapshot rollback)],
 204     },
 205     cpulimit => {
 206         optional => 1,
 207         type => 'number',
 208         description => "Limit of CPU usage.",
 209         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 210         minimum => 0,
 211         maximum => 128,
 212         default => 0,
 213     },
 214     cpuunits => {
 215         optional => 1,
 216         type => 'integer',
 217         description => "CPU weight for a VM.",
 218         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 219         minimum => 0,
 220         maximum => 500000,
 221         default => 1024,
 222     },
 223     memory => {
 224         optional => 1,
 225         type => 'integer',
 226         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 227         minimum => 16,
 228         default => 512,
 229     },
 230     balloon => {
 231         optional => 1,
 232         type => 'integer',
 233         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 234         minimum => 0,
 235     },
 236     shares => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 240         minimum => 0,
 241         maximum => 50000,
 242         default => 1000,
 243     },
 244     keyboard => {
 245         optional => 1,
 246         type => 'string',
 247         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 248         enum => PVE::Tools::kvmkeymaplist(),
 249         default => 'en-us',
 250     },
 251     name => {
 252         optional => 1,
 253         type => 'string', format => 'dns-name',
 254         description => "Set a name for the VM. Only used on the configuration web interface.",
 255     },
 256     scsihw => {
 257         optional => 1,
 258         type => 'string',
 259         description => "SCSI controller model",
 260         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 261         default => 'lsi',
 262     },
 263     description => {
 264         optional => 1,
 265         type => 'string',
 266         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 267     },
 268     ostype => {
 269         optional => 1,
 270         type => 'string',
 271         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 272         description => "Specify guest operating system.",
 273         verbose_description => <<EODESC,
 274 Specify guest operating system. This is used to enable special
 275 optimization/features for specific operating systems:
 276
 277 [horizontal]
 278 other;; unspecified OS
 279 wxp;; Microsoft Windows XP
 280 w2k;; Microsoft Windows 2000
 281 w2k3;; Microsoft Windows 2003
 282 w2k8;; Microsoft Windows 2008
 283 wvista;; Microsoft Windows Vista
 284 win7;; Microsoft Windows 7
 285 win8;; Microsoft Windows 8/2012
 286 l24;; Linux 2.4 Kernel
 287 l26;; Linux 2.6/3.X Kernel
 288 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 289 EODESC
 290     },
 291     boot => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 295         pattern => '[acdn]{1,4}',
 296         default => 'cdn',
 297     },
 298     bootdisk => {
 299         optional => 1,
 300         type => 'string', format => 'pve-qm-bootdisk',
 301         description => "Enable booting from specified disk.",
 302         pattern => '(ide|sata|scsi|virtio)\d+',
 303     },
 304     smp => {
 305         optional => 1,
 306         type => 'integer',
 307         description => "The number of CPUs. Please use option -sockets instead.",
 308         minimum => 1,
 309         default => 1,
 310     },
 311     sockets => {
 312         optional => 1,
 313         type => 'integer',
 314         description => "The number of CPU sockets.",
 315         minimum => 1,
 316         default => 1,
 317     },
 318     cores => {
 319         optional => 1,
 320         type => 'integer',
 321         description => "The number of cores per socket.",
 322         minimum => 1,
 323         default => 1,
 324     },
 325     numa => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Enable/disable NUMA.",
 329         default => 0,
 330     },
 331     hugepages => {
 332         optional => 1,
 333         type => 'string',
 334         description => "Enable/disable hugepages memory.",
 335         enum => [qw(any 2 1024)],
 336     },
 337     vcpus => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Number of hotplugged vcpus.",
 341         minimum => 1,
 342         default => 0,
 343     },
 344     acpi => {
 345         optional => 1,
 346         type => 'boolean',
 347         description => "Enable/disable ACPI.",
 348         default => 1,
 349     },
 350     agent => {
 351         optional => 1,
 352         type => 'boolean',
 353         description => "Enable/disable Qemu GuestAgent.",
 354         default => 0,
 355     },
 356     kvm => {
 357         optional => 1,
 358         type => 'boolean',
 359         description => "Enable/disable KVM hardware virtualization.",
 360         default => 1,
 361     },
 362     tdf => {
 363         optional => 1,
 364         type => 'boolean',
 365         description => "Enable/disable time drift fix.",
 366         default => 0,
 367     },
 368     localtime => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 372     },
 373     freeze => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 377     },
 378     vga => {
 379         optional => 1,
 380         type => 'string',
 381         description => "Select the VGA type.",
 382         verbose_description => "Select the VGA type. If you want to use high resolution" .
 383             " modes (>= 1280x1024x16) then you should use the options " .
 384             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 385             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 386             "display sever. For win* OS you can select how many independent " .
 387             "displays you want, Linux guests can add displays them self. " .
 388             "You can also run without any graphic card, using a serial device" .
 389             " as terminal.",
 390         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 391     },
 392     watchdog => {
 393         optional => 1,
 394         type => 'string', format => 'pve-qm-watchdog',
 395         description => "Create a virtual hardware watchdog device.",
 396         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 397             " (by a guest action), the watchdog must be periodically polled " .
 398             "by an agent inside the guest or else the watchdog will reset " .
 399             "the guest (or execute the respective action specified)",
 400     },
 401     startdate => {
 402         optional => 1,
 403         type => 'string',
 404         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 405         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 406         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 407         default => 'now',
 408     },
 409     startup =>  get_standard_option('pve-startup-order'),
 410     template => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable Template.",
 414         default => 0,
 415     },
 416     args => {
 417         optional => 1,
 418         type => 'string',
 419         description => "Arbitrary arguments passed to kvm.",
 420         verbose_description => <<EODESCR,
 421 Arbitrary arguments passed to kvm, for example:
 422
 423 args: -no-reboot -no-hpet
 424
 425 NOTE: this option is for experts only.
 426 EODESCR
 427     },
 428     tablet => {
 429         optional => 1,
 430         type => 'boolean',
 431         default => 1,
 432         description => "Enable/disable the USB tablet device.",
 433         verbose_description => "Enable/disable the USB tablet device. This device is " .
 434             "usually needed to allow absolute mouse positioning with VNC. " .
 435             "Else the mouse runs out of sync with normal VNC clients. " .
 436             "If you're running lots of console-only guests on one host, " .
 437             "you may consider disabling this to save some context switches. " .
 438             "This is turned off by default if you use spice (-vga=qxl).",
 439     },
 440     migrate_speed => {
 441         optional => 1,
 442         type => 'integer',
 443         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 444         minimum => 0,
 445         default => 0,
 446     },
 447     migrate_downtime => {
 448         optional => 1,
 449         type => 'number',
 450         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 451         minimum => 0,
 452         default => 0.1,
 453     },
 454     cdrom => {
 455         optional => 1,
 456         type => 'string', format => 'pve-qm-ide',
 457         typetext => '<volume>',
 458         description => "This is an alias for option -ide2",
 459     },
 460     cpu => {
 461         optional => 1,
 462         description => "Emulated CPU type.",
 463         type => 'string',
 464         format => $cpu_fmt,
 465     },
 466     parent => get_standard_option('pve-snapshot-name', {
 467         optional => 1,
 468         description => "Parent snapshot name. This is used internally, and should not be modified.",
 469     }),
 470     snaptime => {
 471         optional => 1,
 472         description => "Timestamp for snapshots.",
 473         type => 'integer',
 474         minimum => 0,
 475     },
 476     vmstate => {
 477         optional => 1,
 478         type => 'string', format => 'pve-volume-id',
 479         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 480     },
 481     machine => {
 482         description => "Specific the Qemu machine type.",
 483         type => 'string',
 484         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 485         maxLength => 40,
 486         optional => 1,
 487     },
 488     smbios1 => {
 489         description => "Specify SMBIOS type 1 fields.",
 490         type => 'string', format => 'pve-qm-smbios1',
 491         maxLength => 256,
 492         optional => 1,
 493     },
 494     protection => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 498         default => 0,
 499     },
 500     bios => {
 501         optional => 1,
 502         type => 'string',
 503         enum => [ qw(seabios ovmf) ],
 504         description => "Select BIOS implementation.",
 505         default => 'seabios',
 506     },
 507 };
 508
 509 # what about other qemu settings ?
 510 #cpu => 'string',
 511 #machine => 'string',
 512 #fda => 'file',
 513 #fdb => 'file',
 514 #mtdblock => 'file',
 515 #sd => 'file',
 516 #pflash => 'file',
 517 #snapshot => 'bool',
 518 #bootp => 'file',
 519 ##tftp => 'dir',
 520 ##smb => 'dir',
 521 #kernel => 'file',
 522 #append => 'string',
 523 #initrd => 'file',
 524 ##soundhw => 'string',
 525
 526 while (my ($k, $v) = each %$confdesc) {
 527     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 528 }
 529
 530 my $MAX_IDE_DISKS = 4;
 531 my $MAX_SCSI_DISKS = 14;
 532 my $MAX_VIRTIO_DISKS = 16;
 533 my $MAX_SATA_DISKS = 6;
 534 my $MAX_USB_DEVICES = 5;
 535 my $MAX_NETS = 32;
 536 my $MAX_UNUSED_DISKS = 8;
 537 my $MAX_HOSTPCI_DEVICES = 4;
 538 my $MAX_SERIAL_PORTS = 4;
 539 my $MAX_PARALLEL_PORTS = 3;
 540 my $MAX_NUMA = 8;
 541
 542 my $numa_fmt = {
 543     cpus => {
 544         type => "string",
 545         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 546         description => "CPUs accessing this NUMA node.",
 547         format_description => "id[-id];...",
 548     },
 549     memory => {
 550         type => "number",
 551         description => "Amount of memory this NUMA node provides.",
 552         optional => 1,
 553     },
 554     hostnodes => {
 555         type => "string",
 556         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 557         description => "Host NUMA nodes to use.",
 558         format_description => "id[-id];...",
 559         optional => 1,
 560     },
 561     policy => {
 562         type => 'string',
 563         enum => [qw(preferred bind interleave)],
 564         description => "NUMA allocation policy.",
 565         optional => 1,
 566     },
 567 };
 568 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 569 my $numadesc = {
 570     optional => 1,
 571     type => 'string', format => $numa_fmt,
 572     description => "NUMA topology.",
 573 };
 574 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 575
 576 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 577     $confdesc->{"numa$i"} = $numadesc;
 578 }
 579
 580 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 581                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 582                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 583 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 584
 585 my $net_fmt_bridge_descr = <<__EOD__;
 586 Bridge to attach the network device to. The Proxmox VE standard bridge
 587 is called 'vmbr0'.
 588
 589 If you do not specify a bridge, we create a kvm user (NATed) network
 590 device, which provides DHCP and DNS services. The following addresses
 591 are used:
 592
 593  10.0.2.2   Gateway
 594  10.0.2.3   DNS Server
 595  10.0.2.4   SMB Server
 596
 597 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 598 __EOD__
 599
 600 my $net_fmt = {
 601     macaddr => {
 602         type => 'string',
 603         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 604         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 605         format_description => "XX:XX:XX:XX:XX:XX",
 606         optional => 1,
 607     },
 608     model => {
 609         type => 'string',
 610         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 611         enum => $nic_model_list,
 612         default_key => 1,
 613     },
 614     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 615     bridge => {
 616         type => 'string',
 617         description => $net_fmt_bridge_descr,
 618         format_description => 'bridge',
 619         optional => 1,
 620     },
 621     queues => {
 622         type => 'integer',
 623         minimum => 0, maximum => 16,
 624         description => 'Number of packet queues to be used on the device.',
 625         optional => 1,
 626     },
 627     rate => {
 628         type => 'number',
 629         minimum => 0,
 630         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 631         optional => 1,
 632     },
 633     tag => {
 634         type => 'integer',
 635         minimum => 1, maximum => 4094,
 636         description => 'VLAN tag to apply to packets on this interface.',
 637         optional => 1,
 638     },
 639     trunks => {
 640         type => 'string',
 641         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 642         description => 'VLAN trunks to pass through this interface.',
 643         format_description => 'vlanid[;vlanid...]',
 644         optional => 1,
 645     },
 646     firewall => {
 647         type => 'boolean',
 648         description => 'Whether this interface should be protected by the firewall.',
 649         optional => 1,
 650     },
 651     link_down => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be disconnected (like pulling the plug).',
 654         optional => 1,
 655     },
 656 };
 657
 658 my $netdesc = {
 659     optional => 1,
 660     type => 'string', format => $net_fmt,
 661     description => "Specify network devices.",
 662 };
 663
 664 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 665
 666 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 667     $confdesc->{"net$i"} = $netdesc;
 668 }
 669
 670 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 671 sub verify_volume_id_or_qm_path {
 672     my ($volid, $noerr) = @_;
 673
 674     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 675         return $volid;
 676     }
 677
 678     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 679     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 680     if ($@) {
 681         return undef if $noerr;
 682         die $@;
 683     }
 684     return $volid;
 685 }
 686
 687 my $drivename_hash;
 688
 689 my %drivedesc_base = (
 690     volume => { alias => 'file' },
 691     file => {
 692         type => 'string',
 693         format => 'pve-volume-id-or-qm-path',
 694         default_key => 1,
 695         format_description => 'volume',
 696         description => "The drive's backing volume.",
 697     },
 698     media => {
 699         type => 'string',
 700         enum => [qw(cdrom disk)],
 701         description => "The drive's media type.",
 702         default => 'disk',
 703         optional => 1
 704     },
 705     cyls => {
 706         type => 'integer',
 707         description => "Force the drive's physical geometry to have a specific cylinder count.",
 708         optional => 1
 709     },
 710     heads => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific head count.",
 713         optional => 1
 714     },
 715     secs => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific sector count.",
 718         optional => 1
 719     },
 720     trans => {
 721         type => 'string',
 722         enum => [qw(none lba auto)],
 723         description => "Force disk geometry bios translation mode.",
 724         optional => 1,
 725     },
 726     snapshot => {
 727         type => 'boolean',
 728         description => "Whether the drive should be included when making snapshots.",
 729         optional => 1,
 730     },
 731     cache => {
 732         type => 'string',
 733         enum => [qw(none writethrough writeback unsafe directsync)],
 734         description => "The drive's cache mode",
 735         optional => 1,
 736     },
 737     format => get_standard_option('pve-qm-image-format'),
 738     size => {
 739         type => 'string',
 740         format => 'disk-size',
 741         format_description => 'DiskSize',
 742         description => "Disk size. This is purely informational and has no effect.",
 743         optional => 1,
 744     },
 745     backup => {
 746         type => 'boolean',
 747         description => "Whether the drive should be included when making backups.",
 748         optional => 1,
 749     },
 750     replicate => {
 751         type => 'boolean',
 752         description => 'Whether the drive should considered for replication jobs.',
 753         optional => 1,
 754         default => 1,
 755     },
 756     rerror => {
 757         type => 'string',
 758         enum => [qw(ignore report stop)],
 759         description => 'Read error action.',
 760         optional => 1,
 761     },
 762     werror => {
 763         type => 'string',
 764         enum => [qw(enospc ignore report stop)],
 765         description => 'Write error action.',
 766         optional => 1,
 767     },
 768     aio => {
 769         type => 'string',
 770         enum => [qw(native threads)],
 771         description => 'AIO type to use.',
 772         optional => 1,
 773     },
 774     discard => {
 775         type => 'string',
 776         enum => [qw(ignore on)],
 777         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 778         optional => 1,
 779     },
 780     detect_zeroes => {
 781         type => 'boolean',
 782         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 783         optional => 1,
 784     },
 785     serial => {
 786         type => 'string',
 787         format => 'urlencoded',
 788         format_description => 'serial',
 789         maxLength => 20*3, # *3 since it's %xx url enoded
 790         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 791         optional => 1,
 792     }
 793 );
 794
 795 my %iothread_fmt = ( iothread => {
 796         type => 'boolean',
 797         description => "Whether to use iothreads for this drive",
 798         optional => 1,
 799 });
 800
 801 my %model_fmt = (
 802     model => {
 803         type => 'string',
 804         format => 'urlencoded',
 805         format_description => 'model',
 806         maxLength => 40*3, # *3 since it's %xx url enoded
 807         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 808         optional => 1,
 809     },
 810 );
 811
 812 my %queues_fmt = (
 813     queues => {
 814         type => 'integer',
 815         description => "Number of queues.",
 816         minimum => 2,
 817         optional => 1
 818     }
 819 );
 820
 821 my %scsiblock_fmt = (
 822     scsiblock => {
 823         type => 'boolean',
 824         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 825         optional => 1,
 826         default => 0,
 827     },
 828 );
 829
 830 my $add_throttle_desc = sub {
 831     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 832     my $d = {
 833         type => $type,
 834         format_description => $unit,
 835         description => "Maximum $what in $longunit.",
 836         optional => 1,
 837     };
 838     $d->{minimum} = $minimum if defined($minimum);
 839     $drivedesc_base{$key} = $d;
 840 };
 841 # throughput: (leaky bucket)
 842 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 843 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 844 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 845 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 846 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 847 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 848 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 849 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 850 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 851
 852 # pools: (pool of IO before throttling starts taking effect)
 853 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 854 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 855 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 856 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 857 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 858 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 859
 860 # burst lengths
 861 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 862 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 863 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 864 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 865 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 866 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 867
 868 # legacy support
 869 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 870 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 871 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 872 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 873
 874 my $ide_fmt = {
 875     %drivedesc_base,
 876     %model_fmt,
 877 };
 878 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 879
 880 my $idedesc = {
 881     optional => 1,
 882     type => 'string', format => $ide_fmt,
 883     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 884 };
 885 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 886
 887 my $scsi_fmt = {
 888     %drivedesc_base,
 889     %iothread_fmt,
 890     %queues_fmt,
 891     %scsiblock_fmt,
 892 };
 893 my $scsidesc = {
 894     optional => 1,
 895     type => 'string', format => $scsi_fmt,
 896     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 897 };
 898 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 899
 900 my $sata_fmt = {
 901     %drivedesc_base,
 902 };
 903 my $satadesc = {
 904     optional => 1,
 905     type => 'string', format => $sata_fmt,
 906     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 907 };
 908 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 909
 910 my $virtio_fmt = {
 911     %drivedesc_base,
 912     %iothread_fmt,
 913 };
 914 my $virtiodesc = {
 915     optional => 1,
 916     type => 'string', format => $virtio_fmt,
 917     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 918 };
 919 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 920
 921 my $alldrive_fmt = {
 922     %drivedesc_base,
 923     %iothread_fmt,
 924     %model_fmt,
 925     %queues_fmt,
 926     %scsiblock_fmt,
 927 };
 928
 929 my $efidisk_fmt = {
 930     volume => { alias => 'file' },
 931     file => {
 932         type => 'string',
 933         format => 'pve-volume-id-or-qm-path',
 934         default_key => 1,
 935         format_description => 'volume',
 936         description => "The drive's backing volume.",
 937     },
 938     format => get_standard_option('pve-qm-image-format'),
 939     size => {
 940         type => 'string',
 941         format => 'disk-size',
 942         format_description => 'DiskSize',
 943         description => "Disk size. This is purely informational and has no effect.",
 944         optional => 1,
 945     },
 946 };
 947
 948 my $efidisk_desc = {
 949     optional => 1,
 950     type => 'string', format => $efidisk_fmt,
 951     description => "Configure a Disk for storing EFI vars",
 952 };
 953
 954 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 955
 956 my $usb_fmt = {
 957     host => {
 958         default_key => 1,
 959         type => 'string', format => 'pve-qm-usb-device',
 960         format_description => 'HOSTUSBDEVICE|spice',
 961         description => <<EODESCR,
 962 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 963
 964  'bus-port(.port)*' (decimal numbers) or
 965  'vendor_id:product_id' (hexadeciaml numbers) or
 966  'spice'
 967
 968 You can use the 'lsusb -t' command to list existing usb devices.
 969
 970 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 971
 972 The value 'spice' can be used to add a usb redirection devices for spice.
 973 EODESCR
 974     },
 975     usb3 => {
 976         optional => 1,
 977         type => 'boolean',
 978         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 979         default => 0,
 980     },
 981 };
 982
 983 my $usbdesc = {
 984     optional => 1,
 985     type => 'string', format => $usb_fmt,
 986     description => "Configure an USB device (n is 0 to 4).",
 987 };
 988 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 989
 990 # NOTE: the match-groups of this regex are used in parse_hostpci
 991 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 992 my $hostpci_fmt = {
 993     host => {
 994         default_key => 1,
 995         type => 'string',
 996         pattern => qr/$PCIRE(;$PCIRE)*/,
 997         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 998         description => <<EODESCR,
 999 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1000 of PCI virtual functions of the host. HOSTPCIID syntax is:
1001
1002 'bus:dev.func' (hexadecimal numbers)
1003
1004 You can us the 'lspci' command to list existing PCI devices.
1005 EODESCR
1006     },
1007     rombar => {
1008         type => 'boolean',
1009         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1010         optional => 1,
1011         default => 1,
1012     },
1013     romfile => {
1014         type => 'string',
1015         pattern => '[^,;]+',
1016         format_description => 'string',
1017         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1018         optional => 1,
1019     },
1020     pcie => {
1021         type => 'boolean',
1022         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1023         optional => 1,
1024         default => 0,
1025     },
1026     'x-vga' => {
1027         type => 'boolean',
1028         description =>  "Enable vfio-vga device support.",
1029         optional => 1,
1030         default => 0,
1031     },
1032 };
1033 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1034
1035 my $hostpcidesc = {
1036         optional => 1,
1037         type => 'string', format => 'pve-qm-hostpci',
1038         description => "Map host PCI devices into guest.",
1039         verbose_description =>  <<EODESCR,
1040 Map host PCI devices into guest.
1041
1042 NOTE: This option allows direct access to host hardware. So it is no longer
1043 possible to migrate such machines - use with special care.
1044
1045 CAUTION: Experimental! User reported problems with this option.
1046 EODESCR
1047 };
1048 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1049
1050 my $serialdesc = {
1051         optional => 1,
1052         type => 'string',
1053         pattern => '(/dev/.+|socket)',
1054         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1055         verbose_description =>  <<EODESCR,
1056 Create a serial device inside the VM (n is 0 to 3), and pass through a
1057 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1058 host side (use 'qm terminal' to open a terminal connection).
1059
1060 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1061
1062 CAUTION: Experimental! User reported problems with this option.
1063 EODESCR
1064 };
1065
1066 my $paralleldesc= {
1067         optional => 1,
1068         type => 'string',
1069         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1070         description =>  "Map host parallel devices (n is 0 to 2).",
1071         verbose_description =>  <<EODESCR,
1072 Map host parallel devices (n is 0 to 2).
1073
1074 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1075
1076 CAUTION: Experimental! User reported problems with this option.
1077 EODESCR
1078 };
1079
1080 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1081     $confdesc->{"parallel$i"} = $paralleldesc;
1082 }
1083
1084 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1085     $confdesc->{"serial$i"} = $serialdesc;
1086 }
1087
1088 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1089     $confdesc->{"hostpci$i"} = $hostpcidesc;
1090 }
1091
1092 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1093     $drivename_hash->{"ide$i"} = 1;
1094     $confdesc->{"ide$i"} = $idedesc;
1095 }
1096
1097 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1098     $drivename_hash->{"sata$i"} = 1;
1099     $confdesc->{"sata$i"} = $satadesc;
1100 }
1101
1102 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1103     $drivename_hash->{"scsi$i"} = 1;
1104     $confdesc->{"scsi$i"} = $scsidesc ;
1105 }
1106
1107 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1108     $drivename_hash->{"virtio$i"} = 1;
1109     $confdesc->{"virtio$i"} = $virtiodesc;
1110 }
1111
1112 $drivename_hash->{efidisk0} = 1;
1113 $confdesc->{efidisk0} = $efidisk_desc;
1114
1115 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1116     $confdesc->{"usb$i"} = $usbdesc;
1117 }
1118
1119 my $unuseddesc = {
1120     optional => 1,
1121     type => 'string', format => 'pve-volume-id',
1122     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1123 };
1124
1125 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1126     $confdesc->{"unused$i"} = $unuseddesc;
1127 }
1128
1129 my $kvm_api_version = 0;
1130
1131 sub kvm_version {
1132
1133     return $kvm_api_version if $kvm_api_version;
1134
1135     my $fh = IO::File->new("</dev/kvm") ||
1136         return 0;
1137
1138     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1139         $kvm_api_version = $v;
1140     }
1141
1142     $fh->close();
1143
1144     return  $kvm_api_version;
1145 }
1146
1147 my $kvm_user_version;
1148
1149 sub kvm_user_version {
1150
1151     return $kvm_user_version if $kvm_user_version;
1152
1153     $kvm_user_version = 'unknown';
1154
1155     my $code = sub {
1156         my $line = shift;
1157         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1158             $kvm_user_version = $2;
1159         }
1160     };
1161
1162     eval { run_command("kvm -version", outfunc => $code); };
1163     warn $@ if $@;
1164
1165     return $kvm_user_version;
1166
1167 }
1168
1169 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1170
1171 sub valid_drive_names {
1172     # order is important - used to autoselect boot disk
1173     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1174             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1175             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1176             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1177             'efidisk0');
1178 }
1179
1180 sub is_valid_drivename {
1181     my $dev = shift;
1182
1183     return defined($drivename_hash->{$dev});
1184 }
1185
1186 sub option_exists {
1187     my $key = shift;
1188     return defined($confdesc->{$key});
1189 }
1190
1191 sub nic_models {
1192     return $nic_model_list;
1193 }
1194
1195 sub os_list_description {
1196
1197     return {
1198         other => 'Other',
1199         wxp => 'Windows XP',
1200         w2k => 'Windows 2000',
1201         w2k3 =>, 'Windows 2003',
1202         w2k8 => 'Windows 2008',
1203         wvista => 'Windows Vista',
1204         win7 => 'Windows 7',
1205         win8 => 'Windows 8/2012',
1206         win10 => 'Windows 10/2016',
1207         l24 => 'Linux 2.4',
1208         l26 => 'Linux 2.6',
1209     };
1210 }
1211
1212 my $cdrom_path;
1213
1214 sub get_cdrom_path {
1215
1216     return  $cdrom_path if $cdrom_path;
1217
1218     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1219     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1220     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1221 }
1222
1223 sub get_iso_path {
1224     my ($storecfg, $vmid, $cdrom) = @_;
1225
1226     if ($cdrom eq 'cdrom') {
1227         return get_cdrom_path();
1228     } elsif ($cdrom eq 'none') {
1229         return '';
1230     } elsif ($cdrom =~ m|^/|) {
1231         return $cdrom;
1232     } else {
1233         return PVE::Storage::path($storecfg, $cdrom);
1234     }
1235 }
1236
1237 # try to convert old style file names to volume IDs
1238 sub filename_to_volume_id {
1239     my ($vmid, $file, $media) = @_;
1240
1241     if (!($file eq 'none' || $file eq 'cdrom' ||
1242           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1243
1244         return undef if $file =~ m|/|;
1245
1246         if ($media && $media eq 'cdrom') {
1247             $file = "local:iso/$file";
1248         } else {
1249             $file = "local:$vmid/$file";
1250         }
1251     }
1252
1253     return $file;
1254 }
1255
1256 sub verify_media_type {
1257     my ($opt, $vtype, $media) = @_;
1258
1259     return if !$media;
1260
1261     my $etype;
1262     if ($media eq 'disk') {
1263         $etype = 'images';
1264     } elsif ($media eq 'cdrom') {
1265         $etype = 'iso';
1266     } else {
1267         die "internal error";
1268     }
1269
1270     return if ($vtype eq $etype);
1271
1272     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1273 }
1274
1275 sub cleanup_drive_path {
1276     my ($opt, $storecfg, $drive) = @_;
1277
1278     # try to convert filesystem paths to volume IDs
1279
1280     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1281         ($drive->{file} !~ m|^/dev/.+|) &&
1282         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1283         ($drive->{file} !~ m/^\d+$/)) {
1284         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1285         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1286         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1287         verify_media_type($opt, $vtype, $drive->{media});
1288         $drive->{file} = $volid;
1289     }
1290
1291     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1292 }
1293
1294 sub parse_hotplug_features {
1295     my ($data) = @_;
1296
1297     my $res = {};
1298
1299     return $res if $data eq '0';
1300
1301     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1302
1303     foreach my $feature (PVE::Tools::split_list($data)) {
1304         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1305             $res->{$1} = 1;
1306         } else {
1307             die "invalid hotplug feature '$feature'\n";
1308         }
1309     }
1310     return $res;
1311 }
1312
1313 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1314 sub pve_verify_hotplug_features {
1315     my ($value, $noerr) = @_;
1316
1317     return $value if parse_hotplug_features($value);
1318
1319     return undef if $noerr;
1320
1321     die "unable to parse hotplug option\n";
1322 }
1323
1324 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1325 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1326 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1327 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1328 #        [,iothread=on][,serial=serial][,model=model]
1329
1330 sub parse_drive {
1331     my ($key, $data) = @_;
1332
1333     my ($interface, $index);
1334
1335     if ($key =~ m/^([^\d]+)(\d+)$/) {
1336         $interface = $1;
1337         $index = $2;
1338     } else {
1339         return undef;
1340     }
1341
1342     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1343                                      : $confdesc->{$key}->{format};
1344     if (!$desc) {
1345         warn "invalid drive key: $key\n";
1346         return undef;
1347     }
1348     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1349     return undef if !$res;
1350     $res->{interface} = $interface;
1351     $res->{index} = $index;
1352
1353     my $error = 0;
1354     foreach my $opt (qw(bps bps_rd bps_wr)) {
1355         if (my $bps = defined(delete $res->{$opt})) {
1356             if (defined($res->{"m$opt"})) {
1357                 warn "both $opt and m$opt specified\n";
1358                 ++$error;
1359                 next;
1360             }
1361             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1362         }
1363     }
1364
1365     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1366     for my $requirement (
1367         [mbps_max => 'mbps'],
1368         [mbps_rd_max => 'mbps_rd'],
1369         [mbps_wr_max => 'mbps_wr'],
1370         [miops_max => 'miops'],
1371         [miops_rd_max => 'miops_rd'],
1372         [miops_wr_max => 'miops_wr'],
1373         [bps_max_length => 'mbps_max'],
1374         [bps_rd_max_length => 'mbps_rd_max'],
1375         [bps_wr_max_length => 'mbps_wr_max'],
1376         [iops_max_length => 'iops_max'],
1377         [iops_rd_max_length => 'iops_rd_max'],
1378         [iops_wr_max_length => 'iops_wr_max']) {
1379         my ($option, $requires) = @$requirement;
1380         if ($res->{$option} && !$res->{$requires}) {
1381             warn "$option requires $requires\n";
1382             ++$error;
1383         }
1384     }
1385
1386     return undef if $error;
1387
1388     return undef if $res->{mbps_rd} && $res->{mbps};
1389     return undef if $res->{mbps_wr} && $res->{mbps};
1390     return undef if $res->{iops_rd} && $res->{iops};
1391     return undef if $res->{iops_wr} && $res->{iops};
1392
1393     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1394         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1395         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1396         return undef if $res->{interface} eq 'virtio';
1397     }
1398
1399     if (my $size = $res->{size}) {
1400         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1401     }
1402
1403     return $res;
1404 }
1405
1406 sub print_drive {
1407     my ($vmid, $drive) = @_;
1408     my $data = { %$drive };
1409     delete $data->{$_} for qw(index interface);
1410     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1411 }
1412
1413 sub scsi_inquiry {
1414     my($fh, $noerr) = @_;
1415
1416     my $SG_IO = 0x2285;
1417     my $SG_GET_VERSION_NUM = 0x2282;
1418
1419     my $versionbuf = "\x00" x 8;
1420     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1421     if (!$ret) {
1422         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1423         return undef;
1424     }
1425     my $version = unpack("I", $versionbuf);
1426     if ($version < 30000) {
1427         die "scsi generic interface too old\n"  if !$noerr;
1428         return undef;
1429     }
1430
1431     my $buf = "\x00" x 36;
1432     my $sensebuf = "\x00" x 8;
1433     my $cmd = pack("C x3 C x1", 0x12, 36);
1434
1435     # see /usr/include/scsi/sg.h
1436     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1437
1438     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1439                       length($sensebuf), 0, length($buf), $buf,
1440                       $cmd, $sensebuf, 6000);
1441
1442     $ret = ioctl($fh, $SG_IO, $packet);
1443     if (!$ret) {
1444         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1445         return undef;
1446     }
1447
1448     my @res = unpack($sg_io_hdr_t, $packet);
1449     if ($res[17] || $res[18]) {
1450         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1451         return undef;
1452     }
1453
1454     my $res = {};
1455     (my $byte0, my $byte1, $res->{vendor},
1456      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1457
1458     $res->{removable} = $byte1 & 128 ? 1 : 0;
1459     $res->{type} = $byte0 & 31;
1460
1461     return $res;
1462 }
1463
1464 sub path_is_scsi {
1465     my ($path) = @_;
1466
1467     my $fh = IO::File->new("+<$path") || return undef;
1468     my $res = scsi_inquiry($fh, 1);
1469     close($fh);
1470
1471     return $res;
1472 }
1473
1474 sub machine_type_is_q35 {
1475     my ($conf) = @_;
1476
1477     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1478 }
1479
1480 sub print_tabletdevice_full {
1481     my ($conf) = @_;
1482
1483     my $q35 = machine_type_is_q35($conf);
1484
1485     # we use uhci for old VMs because tablet driver was buggy in older qemu
1486     my $usbbus = $q35 ? "ehci" : "uhci";
1487
1488     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1489 }
1490
1491 sub print_drivedevice_full {
1492     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1493
1494     my $device = '';
1495     my $maxdev = 0;
1496
1497     if ($drive->{interface} eq 'virtio') {
1498         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1499         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1500         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1501     } elsif ($drive->{interface} eq 'scsi') {
1502
1503         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1504         my $unit = $drive->{index} % $maxdev;
1505         my $devicetype = 'hd';
1506         my $path = '';
1507         if (drive_is_cdrom($drive)) {
1508             $devicetype = 'cd';
1509         } else {
1510             if ($drive->{file} =~ m|^/|) {
1511                 $path = $drive->{file};
1512                 if (my $info = path_is_scsi($path)) {
1513                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1514                         $devicetype = 'block';
1515                     } elsif ($info->{type} == 1) { # tape
1516                         $devicetype = 'generic';
1517                     }
1518                 }
1519             } else {
1520                  $path = PVE::Storage::path($storecfg, $drive->{file});
1521             }
1522
1523             if($path =~ m/^iscsi\:\/\//){
1524                 $devicetype = 'generic';
1525             }
1526         }
1527
1528         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1529             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1530         } else {
1531             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1532         }
1533
1534     } elsif ($drive->{interface} eq 'ide'){
1535         $maxdev = 2;
1536         my $controller = int($drive->{index} / $maxdev);
1537         my $unit = $drive->{index} % $maxdev;
1538         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1539
1540         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1541         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1542             $model = URI::Escape::uri_unescape($model);
1543             $device .= ",model=$model";
1544         }
1545     } elsif ($drive->{interface} eq 'sata'){
1546         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1547         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1548         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1549     } elsif ($drive->{interface} eq 'usb') {
1550         die "implement me";
1551         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1552     } else {
1553         die "unsupported interface type";
1554     }
1555
1556     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1557
1558     return $device;
1559 }
1560
1561 sub get_initiator_name {
1562     my $initiator;
1563
1564     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1565     while (defined(my $line = <$fh>)) {
1566         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1567         $initiator = $1;
1568         last;
1569     }
1570     $fh->close();
1571
1572     return $initiator;
1573 }
1574
1575 sub print_drive_full {
1576     my ($storecfg, $vmid, $drive) = @_;
1577
1578     my $path;
1579     my $volid = $drive->{file};
1580     my $format;
1581
1582     if (drive_is_cdrom($drive)) {
1583         $path = get_iso_path($storecfg, $vmid, $volid);
1584     } else {
1585         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1586         if ($storeid) {
1587             $path = PVE::Storage::path($storecfg, $volid);
1588             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1589             $format = qemu_img_format($scfg, $volname);
1590         } else {
1591             $path = $volid;
1592             $format = "raw";
1593         }
1594    }
1595
1596     my $opts = '';
1597     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1598     foreach my $o (@qemu_drive_options) {
1599         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1600     }
1601     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1602         my ($dir, $qmpname) = @$type;
1603         if (my $v = $drive->{"mbps$dir"}) {
1604             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1605         }
1606         if (my $v = $drive->{"mbps${dir}_max"}) {
1607             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1608         }
1609         if (my $v = $drive->{"bps${dir}_max_length"}) {
1610             $opts .= ",throttling.bps$qmpname-max-length=$v";
1611         }
1612         if (my $v = $drive->{"iops${dir}"}) {
1613             $opts .= ",throttling.iops$qmpname=$v";
1614         }
1615         if (my $v = $drive->{"iops${dir}_max"}) {
1616             $opts .= ",throttling.iops$qmpname-max=$v";
1617         }
1618         if (my $v = $drive->{"iops${dir}_max_length"}) {
1619             $opts .= ",throttling.iops$qmpname-max-length=$v";
1620         }
1621     }
1622
1623     if (my $serial = $drive->{serial}) {
1624         $serial = URI::Escape::uri_unescape($serial);
1625         $opts .= ",serial=$serial";
1626     }
1627
1628     $opts .= ",format=$format" if $format && !$drive->{format};
1629
1630     my $cache_direct = 0;
1631
1632     if (my $cache = $drive->{cache}) {
1633         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1634     } elsif (!drive_is_cdrom($drive)) {
1635         $opts .= ",cache=none";
1636         $cache_direct = 1;
1637     }
1638
1639     # aio native works only with O_DIRECT
1640     if (!$drive->{aio}) {
1641         if($cache_direct) {
1642             $opts .= ",aio=native";
1643         } else {
1644             $opts .= ",aio=threads";
1645         }
1646     }
1647
1648     if (!drive_is_cdrom($drive)) {
1649         my $detectzeroes;
1650         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1651             $detectzeroes = 'off';
1652         } elsif ($drive->{discard}) {
1653             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1654         } else {
1655             # This used to be our default with discard not being specified:
1656             $detectzeroes = 'on';
1657         }
1658         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1659     }
1660
1661     my $pathinfo = $path ? "file=$path," : '';
1662
1663     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1664 }
1665
1666 sub print_netdevice_full {
1667     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1668
1669     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1670
1671     my $device = $net->{model};
1672     if ($net->{model} eq 'virtio') {
1673          $device = 'virtio-net-pci';
1674      };
1675
1676     my $pciaddr = print_pci_addr("$netid", $bridges);
1677     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1678     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1679         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1680         my $vectors = $net->{queues} * 2 + 2;
1681         $tmpstr .= ",vectors=$vectors,mq=on";
1682     }
1683     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1684
1685     if ($use_old_bios_files) {
1686         my $romfile;
1687         if ($device eq 'virtio-net-pci') {
1688             $romfile = 'pxe-virtio.rom';
1689         } elsif ($device eq 'e1000') {
1690             $romfile = 'pxe-e1000.rom';
1691         } elsif ($device eq 'ne2k') {
1692             $romfile = 'pxe-ne2k_pci.rom';
1693         } elsif ($device eq 'pcnet') {
1694             $romfile = 'pxe-pcnet.rom';
1695         } elsif ($device eq 'rtl8139') {
1696             $romfile = 'pxe-rtl8139.rom';
1697         }
1698         $tmpstr .= ",romfile=$romfile" if $romfile;
1699     }
1700
1701     return $tmpstr;
1702 }
1703
1704 sub print_netdev_full {
1705     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1706
1707     my $i = '';
1708     if ($netid =~ m/^net(\d+)$/) {
1709         $i = int($1);
1710     }
1711
1712     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1713
1714     my $ifname = "tap${vmid}i$i";
1715
1716     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1717     die "interface name '$ifname' is too long (max 15 character)\n"
1718         if length($ifname) >= 16;
1719
1720     my $vhostparam = '';
1721     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1722
1723     my $vmname = $conf->{name} || "vm$vmid";
1724
1725     my $netdev = "";
1726     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1727
1728     if ($net->{bridge}) {
1729         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1730     } else {
1731         $netdev = "type=user,id=$netid,hostname=$vmname";
1732     }
1733
1734     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1735
1736     return $netdev;
1737 }
1738
1739
1740 sub print_cpu_device {
1741     my ($conf, $id) = @_;
1742
1743     my $kvm = $conf->{kvm} // 1;
1744     my $cpu = $kvm ? "kvm64" : "qemu64";
1745     if (my $cputype = $conf->{cpu}) {
1746         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1747             or die "Cannot parse cpu description: $cputype\n";
1748         $cpu = $cpuconf->{cputype};
1749     }
1750
1751     my $cores = $conf->{cores} || 1;
1752
1753     my $current_core = ($id - 1) % $cores;
1754     my $current_socket = int(($id - 1 - $current_core)/$cores);
1755
1756     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1757 }
1758
1759 sub drive_is_cdrom {
1760     my ($drive) = @_;
1761
1762     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1763
1764 }
1765
1766 sub parse_number_sets {
1767     my ($set) = @_;
1768     my $res = [];
1769     foreach my $part (split(/;/, $set)) {
1770         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1771             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1772             push @$res, [ $1, $2 ];
1773         } else {
1774             die "invalid range: $part\n";
1775         }
1776     }
1777     return $res;
1778 }
1779
1780 sub parse_numa {
1781     my ($data) = @_;
1782
1783     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1784     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1785     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1786     return $res;
1787 }
1788
1789 sub parse_hostpci {
1790     my ($value) = @_;
1791
1792     return undef if !$value;
1793
1794     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1795
1796     my @idlist = split(/;/, $res->{host});
1797     delete $res->{host};
1798     foreach my $id (@idlist) {
1799         if ($id =~ /^$PCIRE$/) {
1800             if (defined($2)) {
1801                 push @{$res->{pciid}}, { id => $1, function => $2 };
1802             } else {
1803                 my $pcidevices = lspci($1);
1804                 $res->{pciid} = $pcidevices->{$1};
1805             }
1806         } else {
1807             # should have been caught by parse_property_string already
1808             die "failed to parse PCI id: $id\n";
1809         }
1810     }
1811     return $res;
1812 }
1813
1814 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1815 sub parse_net {
1816     my ($data) = @_;
1817
1818     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1819     if ($@) {
1820         warn $@;
1821         return undef;
1822     }
1823     if (!defined($res->{macaddr})) {
1824         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1825         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1826     }
1827     return $res;
1828 }
1829
1830 sub print_net {
1831     my $net = shift;
1832
1833     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1834 }
1835
1836 sub add_random_macs {
1837     my ($settings) = @_;
1838
1839     foreach my $opt (keys %$settings) {
1840         next if $opt !~ m/^net(\d+)$/;
1841         my $net = parse_net($settings->{$opt});
1842         next if !$net;
1843         $settings->{$opt} = print_net($net);
1844     }
1845 }
1846
1847 sub vm_is_volid_owner {
1848     my ($storecfg, $vmid, $volid) = @_;
1849
1850     if ($volid !~  m|^/|) {
1851         my ($path, $owner);
1852         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1853         if ($owner && ($owner == $vmid)) {
1854             return 1;
1855         }
1856     }
1857
1858     return undef;
1859 }
1860
1861 sub split_flagged_list {
1862     my $text = shift || '';
1863     $text =~ s/[,;]/ /g;
1864     $text =~ s/^\s+//;
1865     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1866 }
1867
1868 sub join_flagged_list {
1869     my ($how, $lst) = @_;
1870     join $how, map { $lst->{$_} . $_ } keys %$lst;
1871 }
1872
1873 sub vmconfig_delete_pending_option {
1874     my ($conf, $key, $force) = @_;
1875
1876     delete $conf->{pending}->{$key};
1877     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1878     $pending_delete_hash->{$key} = $force ? '!' : '';
1879     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1880 }
1881
1882 sub vmconfig_undelete_pending_option {
1883     my ($conf, $key) = @_;
1884
1885     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1886     delete $pending_delete_hash->{$key};
1887
1888     if (%$pending_delete_hash) {
1889         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1890     } else {
1891         delete $conf->{pending}->{delete};
1892     }
1893 }
1894
1895 sub vmconfig_register_unused_drive {
1896     my ($storecfg, $vmid, $conf, $drive) = @_;
1897
1898     if (!drive_is_cdrom($drive)) {
1899         my $volid = $drive->{file};
1900         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1901             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1902         }
1903     }
1904 }
1905
1906 sub vmconfig_cleanup_pending {
1907     my ($conf) = @_;
1908
1909     # remove pending changes when nothing changed
1910     my $changes;
1911     foreach my $opt (keys %{$conf->{pending}}) {
1912         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1913             $changes = 1;
1914             delete $conf->{pending}->{$opt};
1915         }
1916     }
1917
1918     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1919     my $pending_delete_hash = {};
1920     while (my ($opt, $force) = each %$current_delete_hash) {
1921         if (defined($conf->{$opt})) {
1922             $pending_delete_hash->{$opt} = $force;
1923         } else {
1924             $changes = 1;
1925         }
1926     }
1927
1928     if (%$pending_delete_hash) {
1929         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1930     } else {
1931         delete $conf->{pending}->{delete};
1932     }
1933
1934     return $changes;
1935 }
1936
1937 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1938 my $smbios1_fmt = {
1939     uuid => {
1940         type => 'string',
1941         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1942         format_description => 'UUID',
1943         description => "Set SMBIOS1 UUID.",
1944         optional => 1,
1945     },
1946     version => {
1947         type => 'string',
1948         pattern => '\S+',
1949         format_description => 'string',
1950         description => "Set SMBIOS1 version.",
1951         optional => 1,
1952     },
1953     serial => {
1954         type => 'string',
1955         pattern => '\S+',
1956         format_description => 'string',
1957         description => "Set SMBIOS1 serial number.",
1958         optional => 1,
1959     },
1960     manufacturer => {
1961         type => 'string',
1962         pattern => '\S+',
1963         format_description => 'string',
1964         description => "Set SMBIOS1 manufacturer.",
1965         optional => 1,
1966     },
1967     product => {
1968         type => 'string',
1969         pattern => '\S+',
1970         format_description => 'string',
1971         description => "Set SMBIOS1 product ID.",
1972         optional => 1,
1973     },
1974     sku => {
1975         type => 'string',
1976         pattern => '\S+',
1977         format_description => 'string',
1978         description => "Set SMBIOS1 SKU string.",
1979         optional => 1,
1980     },
1981     family => {
1982         type => 'string',
1983         pattern => '\S+',
1984         format_description => 'string',
1985         description => "Set SMBIOS1 family string.",
1986         optional => 1,
1987     },
1988 };
1989
1990 sub parse_smbios1 {
1991     my ($data) = @_;
1992
1993     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1994     warn $@ if $@;
1995     return $res;
1996 }
1997
1998 sub print_smbios1 {
1999     my ($smbios1) = @_;
2000     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2001 }
2002
2003 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2004
2005 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2006 sub verify_bootdisk {
2007     my ($value, $noerr) = @_;
2008
2009     return $value if is_valid_drivename($value);
2010
2011     return undef if $noerr;
2012
2013     die "invalid boot disk '$value'\n";
2014 }
2015
2016 sub parse_watchdog {
2017     my ($value) = @_;
2018
2019     return undef if !$value;
2020
2021     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2022     warn $@ if $@;
2023     return $res;
2024 }
2025
2026 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2027 sub verify_usb_device {
2028     my ($value, $noerr) = @_;
2029
2030     return $value if parse_usb_device($value);
2031
2032     return undef if $noerr;
2033
2034     die "unable to parse usb device\n";
2035 }
2036
2037 # add JSON properties for create and set function
2038 sub json_config_properties {
2039     my $prop = shift;
2040
2041     foreach my $opt (keys %$confdesc) {
2042         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2043         $prop->{$opt} = $confdesc->{$opt};
2044     }
2045
2046     return $prop;
2047 }
2048
2049 sub check_type {
2050     my ($key, $value) = @_;
2051
2052     die "unknown setting '$key'\n" if !$confdesc->{$key};
2053
2054     my $type = $confdesc->{$key}->{type};
2055
2056     if (!defined($value)) {
2057         die "got undefined value\n";
2058     }
2059
2060     if ($value =~ m/[\n\r]/) {
2061         die "property contains a line feed\n";
2062     }
2063
2064     if ($type eq 'boolean') {
2065         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2066         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2067         die "type check ('boolean') failed - got '$value'\n";
2068     } elsif ($type eq 'integer') {
2069         return int($1) if $value =~ m/^(\d+)$/;
2070         die "type check ('integer') failed - got '$value'\n";
2071     } elsif ($type eq 'number') {
2072         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2073         die "type check ('number') failed - got '$value'\n";
2074     } elsif ($type eq 'string') {
2075         if (my $fmt = $confdesc->{$key}->{format}) {
2076             PVE::JSONSchema::check_format($fmt, $value);
2077             return $value;
2078         }
2079         $value =~ s/^\"(.*)\"$/$1/;
2080         return $value;
2081     } else {
2082         die "internal error"
2083     }
2084 }
2085
2086 sub check_iommu_support{
2087     #fixme : need to check IOMMU support
2088     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2089
2090     my $iommu=1;
2091     return $iommu;
2092
2093 }
2094
2095 sub touch_config {
2096     my ($vmid) = @_;
2097
2098     my $conf = PVE::QemuConfig->config_file($vmid);
2099     utime undef, undef, $conf;
2100 }
2101
2102 sub destroy_vm {
2103     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2104
2105     my $conffile = PVE::QemuConfig->config_file($vmid);
2106
2107     my $conf = PVE::QemuConfig->load_config($vmid);
2108
2109     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2110
2111     # only remove disks owned by this VM
2112     foreach_drive($conf, sub {
2113         my ($ds, $drive) = @_;
2114
2115         return if drive_is_cdrom($drive);
2116
2117         my $volid = $drive->{file};
2118
2119         return if !$volid || $volid =~ m|^/|;
2120
2121         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2122         return if !$path || !$owner || ($owner != $vmid);
2123
2124         eval {
2125             PVE::Storage::vdisk_free($storecfg, $volid);
2126         };
2127         warn "Could not remove disk '$volid', check manually: $@" if $@;
2128
2129     });
2130
2131     if ($keep_empty_config) {
2132         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2133     } else {
2134         unlink $conffile;
2135     }
2136
2137     # also remove unused disk
2138     eval {
2139         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2140
2141         eval {
2142             PVE::Storage::foreach_volid($dl, sub {
2143                 my ($volid, $sid, $volname, $d) = @_;
2144                 PVE::Storage::vdisk_free($storecfg, $volid);
2145             });
2146         };
2147         warn $@ if $@;
2148
2149     };
2150     warn $@ if $@;
2151 }
2152
2153 sub parse_vm_config {
2154     my ($filename, $raw) = @_;
2155
2156     return undef if !defined($raw);
2157
2158     my $res = {
2159         digest => Digest::SHA::sha1_hex($raw),
2160         snapshots => {},
2161         pending => {},
2162     };
2163
2164     $filename =~ m|/qemu-server/(\d+)\.conf$|
2165         || die "got strange filename '$filename'";
2166
2167     my $vmid = $1;
2168
2169     my $conf = $res;
2170     my $descr;
2171     my $section = '';
2172
2173     my @lines = split(/\n/, $raw);
2174     foreach my $line (@lines) {
2175         next if $line =~ m/^\s*$/;
2176
2177         if ($line =~ m/^\[PENDING\]\s*$/i) {
2178             $section = 'pending';
2179             if (defined($descr)) {
2180                 $descr =~ s/\s+$//;
2181                 $conf->{description} = $descr;
2182             }
2183             $descr = undef;
2184             $conf = $res->{$section} = {};
2185             next;
2186
2187         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2188             $section = $1;
2189             if (defined($descr)) {
2190                 $descr =~ s/\s+$//;
2191                 $conf->{description} = $descr;
2192             }
2193             $descr = undef;
2194             $conf = $res->{snapshots}->{$section} = {};
2195             next;
2196         }
2197
2198         if ($line =~ m/^\#(.*)\s*$/) {
2199             $descr = '' if !defined($descr);
2200             $descr .= PVE::Tools::decode_text($1) . "\n";
2201             next;
2202         }
2203
2204         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2205             $descr = '' if !defined($descr);
2206             $descr .= PVE::Tools::decode_text($2);
2207         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2208             $conf->{snapstate} = $1;
2209         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2210             my $key = $1;
2211             my $value = $2;
2212             $conf->{$key} = $value;
2213         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2214             my $value = $1;
2215             if ($section eq 'pending') {
2216                 $conf->{delete} = $value; # we parse this later
2217             } else {
2218                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2219             }
2220         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2221             my $key = $1;
2222             my $value = $2;
2223             eval { $value = check_type($key, $value); };
2224             if ($@) {
2225                 warn "vm $vmid - unable to parse value of '$key' - $@";
2226             } else {
2227                 $key = 'ide2' if $key eq 'cdrom';
2228                 my $fmt = $confdesc->{$key}->{format};
2229                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2230                     my $v = parse_drive($key, $value);
2231                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2232                         $v->{file} = $volid;
2233                         $value = print_drive($vmid, $v);
2234                     } else {
2235                         warn "vm $vmid - unable to parse value of '$key'\n";
2236                         next;
2237                     }
2238                 }
2239
2240                 $conf->{$key} = $value;
2241             }
2242         }
2243     }
2244
2245     if (defined($descr)) {
2246         $descr =~ s/\s+$//;
2247         $conf->{description} = $descr;
2248     }
2249     delete $res->{snapstate}; # just to be sure
2250
2251     return $res;
2252 }
2253
2254 sub write_vm_config {
2255     my ($filename, $conf) = @_;
2256
2257     delete $conf->{snapstate}; # just to be sure
2258
2259     if ($conf->{cdrom}) {
2260         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2261         $conf->{ide2} = $conf->{cdrom};
2262         delete $conf->{cdrom};
2263     }
2264
2265     # we do not use 'smp' any longer
2266     if ($conf->{sockets}) {
2267         delete $conf->{smp};
2268     } elsif ($conf->{smp}) {
2269         $conf->{sockets} = $conf->{smp};
2270         delete $conf->{cores};
2271         delete $conf->{smp};
2272     }
2273
2274     my $used_volids = {};
2275
2276     my $cleanup_config = sub {
2277         my ($cref, $pending, $snapname) = @_;
2278
2279         foreach my $key (keys %$cref) {
2280             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2281                 $key eq 'snapstate' || $key eq 'pending';
2282             my $value = $cref->{$key};
2283             if ($key eq 'delete') {
2284                 die "propertry 'delete' is only allowed in [PENDING]\n"
2285                     if !$pending;
2286                 # fixme: check syntax?
2287                 next;
2288             }
2289             eval { $value = check_type($key, $value); };
2290             die "unable to parse value of '$key' - $@" if $@;
2291
2292             $cref->{$key} = $value;
2293
2294             if (!$snapname && is_valid_drivename($key)) {
2295                 my $drive = parse_drive($key, $value);
2296                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2297             }
2298         }
2299     };
2300
2301     &$cleanup_config($conf);
2302
2303     &$cleanup_config($conf->{pending}, 1);
2304
2305     foreach my $snapname (keys %{$conf->{snapshots}}) {
2306         die "internal error" if $snapname eq 'pending';
2307         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2308     }
2309
2310     # remove 'unusedX' settings if we re-add a volume
2311     foreach my $key (keys %$conf) {
2312         my $value = $conf->{$key};
2313         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2314             delete $conf->{$key};
2315         }
2316     }
2317
2318     my $generate_raw_config = sub {
2319         my ($conf, $pending) = @_;
2320
2321         my $raw = '';
2322
2323         # add description as comment to top of file
2324         if (defined(my $descr = $conf->{description})) {
2325             if ($descr) {
2326                 foreach my $cl (split(/\n/, $descr)) {
2327                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2328                 }
2329             } else {
2330                 $raw .= "#\n" if $pending;
2331             }
2332         }
2333
2334         foreach my $key (sort keys %$conf) {
2335             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2336             $raw .= "$key: $conf->{$key}\n";
2337         }
2338         return $raw;
2339     };
2340
2341     my $raw = &$generate_raw_config($conf);
2342
2343     if (scalar(keys %{$conf->{pending}})){
2344         $raw .= "\n[PENDING]\n";
2345         $raw .= &$generate_raw_config($conf->{pending}, 1);
2346     }
2347
2348     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2349         $raw .= "\n[$snapname]\n";
2350         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2351     }
2352
2353     return $raw;
2354 }
2355
2356 sub load_defaults {
2357
2358     my $res = {};
2359
2360     # we use static defaults from our JSON schema configuration
2361     foreach my $key (keys %$confdesc) {
2362         if (defined(my $default = $confdesc->{$key}->{default})) {
2363             $res->{$key} = $default;
2364         }
2365     }
2366
2367     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2368     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2369
2370     return $res;
2371 }
2372
2373 sub config_list {
2374     my $vmlist = PVE::Cluster::get_vmlist();
2375     my $res = {};
2376     return $res if !$vmlist || !$vmlist->{ids};
2377     my $ids = $vmlist->{ids};
2378
2379     foreach my $vmid (keys %$ids) {
2380         my $d = $ids->{$vmid};
2381         next if !$d->{node} || $d->{node} ne $nodename;
2382         next if !$d->{type} || $d->{type} ne 'qemu';
2383         $res->{$vmid}->{exists} = 1;
2384     }
2385     return $res;
2386 }
2387
2388 # test if VM uses local resources (to prevent migration)
2389 sub check_local_resources {
2390     my ($conf, $noerr) = @_;
2391
2392     my $loc_res = 0;
2393
2394     $loc_res = 1 if $conf->{hostusb}; # old syntax
2395     $loc_res = 1 if $conf->{hostpci}; # old syntax
2396
2397     foreach my $k (keys %$conf) {
2398         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2399         # sockets are safe: they will recreated be on the target side post-migrate
2400         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2401         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2402     }
2403
2404     die "VM uses local resources\n" if $loc_res && !$noerr;
2405
2406     return $loc_res;
2407 }
2408
2409 # check if used storages are available on all nodes (use by migrate)
2410 sub check_storage_availability {
2411     my ($storecfg, $conf, $node) = @_;
2412
2413     foreach_drive($conf, sub {
2414         my ($ds, $drive) = @_;
2415
2416         my $volid = $drive->{file};
2417         return if !$volid;
2418
2419         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2420         return if !$sid;
2421
2422         # check if storage is available on both nodes
2423         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2424         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2425    });
2426 }
2427
2428 # list nodes where all VM images are available (used by has_feature API)
2429 sub shared_nodes {
2430     my ($conf, $storecfg) = @_;
2431
2432     my $nodelist = PVE::Cluster::get_nodelist();
2433     my $nodehash = { map { $_ => 1 } @$nodelist };
2434     my $nodename = PVE::INotify::nodename();
2435
2436     foreach_drive($conf, sub {
2437         my ($ds, $drive) = @_;
2438
2439         my $volid = $drive->{file};
2440         return if !$volid;
2441
2442         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2443         if ($storeid) {
2444             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2445             if ($scfg->{disable}) {
2446                 $nodehash = {};
2447             } elsif (my $avail = $scfg->{nodes}) {
2448                 foreach my $node (keys %$nodehash) {
2449                     delete $nodehash->{$node} if !$avail->{$node};
2450                 }
2451             } elsif (!$scfg->{shared}) {
2452                 foreach my $node (keys %$nodehash) {
2453                     delete $nodehash->{$node} if $node ne $nodename
2454                 }
2455             }
2456         }
2457     });
2458
2459     return $nodehash
2460 }
2461
2462 sub check_cmdline {
2463     my ($pidfile, $pid) = @_;
2464
2465     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2466     if (defined($fh)) {
2467         my $line = <$fh>;
2468         $fh->close;
2469         return undef if !$line;
2470         my @param = split(/\0/, $line);
2471
2472         my $cmd = $param[0];
2473         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2474
2475         for (my $i = 0; $i < scalar (@param); $i++) {
2476             my $p = $param[$i];
2477             next if !$p;
2478             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2479                 my $p = $param[$i+1];
2480                 return 1 if $p && ($p eq $pidfile);
2481                 return undef;
2482             }
2483         }
2484     }
2485     return undef;
2486 }
2487
2488 sub check_running {
2489     my ($vmid, $nocheck, $node) = @_;
2490
2491     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2492
2493     die "unable to find configuration file for VM $vmid - no such machine\n"
2494         if !$nocheck && ! -f $filename;
2495
2496     my $pidfile = pidfile_name($vmid);
2497
2498     if (my $fd = IO::File->new("<$pidfile")) {
2499         my $st = stat($fd);
2500         my $line = <$fd>;
2501         close($fd);
2502
2503         my $mtime = $st->mtime;
2504         if ($mtime > time()) {
2505             warn "file '$filename' modified in future\n";
2506         }
2507
2508         if ($line =~ m/^(\d+)$/) {
2509             my $pid = $1;
2510             if (check_cmdline($pidfile, $pid)) {
2511                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2512                     return $pid;
2513                 }
2514             }
2515         }
2516     }
2517
2518     return undef;
2519 }
2520
2521 sub vzlist {
2522
2523     my $vzlist = config_list();
2524
2525     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2526
2527     while (defined(my $de = $fd->read)) {
2528         next if $de !~ m/^(\d+)\.pid$/;
2529         my $vmid = $1;
2530         next if !defined($vzlist->{$vmid});
2531         if (my $pid = check_running($vmid)) {
2532             $vzlist->{$vmid}->{pid} = $pid;
2533         }
2534     }
2535
2536     return $vzlist;
2537 }
2538
2539 sub disksize {
2540     my ($storecfg, $conf) = @_;
2541
2542     my $bootdisk = $conf->{bootdisk};
2543     return undef if !$bootdisk;
2544     return undef if !is_valid_drivename($bootdisk);
2545
2546     return undef if !$conf->{$bootdisk};
2547
2548     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2549     return undef if !defined($drive);
2550
2551     return undef if drive_is_cdrom($drive);
2552
2553     my $volid = $drive->{file};
2554     return undef if !$volid;
2555
2556     return $drive->{size};
2557 }
2558
2559 my $last_proc_pid_stat;
2560
2561 # get VM status information
2562 # This must be fast and should not block ($full == false)
2563 # We only query KVM using QMP if $full == true (this can be slow)
2564 sub vmstatus {
2565     my ($opt_vmid, $full) = @_;
2566
2567     my $res = {};
2568
2569     my $storecfg = PVE::Storage::config();
2570
2571     my $list = vzlist();
2572     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2573
2574     my $cpucount = $cpuinfo->{cpus} || 1;
2575
2576     foreach my $vmid (keys %$list) {
2577         next if $opt_vmid && ($vmid ne $opt_vmid);
2578
2579         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2580         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2581
2582         my $d = {};
2583         $d->{pid} = $list->{$vmid}->{pid};
2584
2585         # fixme: better status?
2586         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2587
2588         my $size = disksize($storecfg, $conf);
2589         if (defined($size)) {
2590             $d->{disk} = 0; # no info available
2591             $d->{maxdisk} = $size;
2592         } else {
2593             $d->{disk} = 0;
2594             $d->{maxdisk} = 0;
2595         }
2596
2597         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2598         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2599         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2600
2601         $d->{name} = $conf->{name} || "VM $vmid";
2602         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2603
2604         if ($conf->{balloon}) {
2605             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2606             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2607         }
2608
2609         $d->{uptime} = 0;
2610         $d->{cpu} = 0;
2611         $d->{mem} = 0;
2612
2613         $d->{netout} = 0;
2614         $d->{netin} = 0;
2615
2616         $d->{diskread} = 0;
2617         $d->{diskwrite} = 0;
2618
2619         $d->{template} = PVE::QemuConfig->is_template($conf);
2620
2621         $res->{$vmid} = $d;
2622     }
2623
2624     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2625     foreach my $dev (keys %$netdev) {
2626         next if $dev !~ m/^tap([1-9]\d*)i/;
2627         my $vmid = $1;
2628         my $d = $res->{$vmid};
2629         next if !$d;
2630
2631         $d->{netout} += $netdev->{$dev}->{receive};
2632         $d->{netin} += $netdev->{$dev}->{transmit};
2633
2634         if ($full) {
2635             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2636             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2637         }
2638
2639     }
2640
2641     my $ctime = gettimeofday;
2642
2643     foreach my $vmid (keys %$list) {
2644
2645         my $d = $res->{$vmid};
2646         my $pid = $d->{pid};
2647         next if !$pid;
2648
2649         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2650         next if !$pstat; # not running
2651
2652         my $used = $pstat->{utime} + $pstat->{stime};
2653
2654         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2655
2656         if ($pstat->{vsize}) {
2657             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2658         }
2659
2660         my $old = $last_proc_pid_stat->{$pid};
2661         if (!$old) {
2662             $last_proc_pid_stat->{$pid} = {
2663                 time => $ctime,
2664                 used => $used,
2665                 cpu => 0,
2666             };
2667             next;
2668         }
2669
2670         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2671
2672         if ($dtime > 1000) {
2673             my $dutime = $used -  $old->{used};
2674
2675             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2676             $last_proc_pid_stat->{$pid} = {
2677                 time => $ctime,
2678                 used => $used,
2679                 cpu => $d->{cpu},
2680             };
2681         } else {
2682             $d->{cpu} = $old->{cpu};
2683         }
2684     }
2685
2686     return $res if !$full;
2687
2688     my $qmpclient = PVE::QMPClient->new();
2689
2690     my $ballooncb = sub {
2691         my ($vmid, $resp) = @_;
2692
2693         my $info = $resp->{'return'};
2694         return if !$info->{max_mem};
2695
2696         my $d = $res->{$vmid};
2697
2698         # use memory assigned to VM
2699         $d->{maxmem} = $info->{max_mem};
2700         $d->{balloon} = $info->{actual};
2701
2702         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2703             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2704             $d->{freemem} = $info->{free_mem};
2705         }
2706
2707         $d->{ballooninfo} = $info;
2708     };
2709
2710     my $blockstatscb = sub {
2711         my ($vmid, $resp) = @_;
2712         my $data = $resp->{'return'} || [];
2713         my $totalrdbytes = 0;
2714         my $totalwrbytes = 0;
2715
2716         for my $blockstat (@$data) {
2717             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2718             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2719
2720             $blockstat->{device} =~ s/drive-//;
2721             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2722         }
2723         $res->{$vmid}->{diskread} = $totalrdbytes;
2724         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2725     };
2726
2727     my $statuscb = sub {
2728         my ($vmid, $resp) = @_;
2729
2730         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2731         # this fails if ballon driver is not loaded, so this must be
2732         # the last commnand (following command are aborted if this fails).
2733         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2734
2735         my $status = 'unknown';
2736         if (!defined($status = $resp->{'return'}->{status})) {
2737             warn "unable to get VM status\n";
2738             return;
2739         }
2740
2741         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2742     };
2743
2744     foreach my $vmid (keys %$list) {
2745         next if $opt_vmid && ($vmid ne $opt_vmid);
2746         next if !$res->{$vmid}->{pid}; # not running
2747         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2748     }
2749
2750     $qmpclient->queue_execute(undef, 2);
2751
2752     foreach my $vmid (keys %$list) {
2753         next if $opt_vmid && ($vmid ne $opt_vmid);
2754         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2755     }
2756
2757     return $res;
2758 }
2759
2760 sub foreach_drive {
2761     my ($conf, $func, @param) = @_;
2762
2763     foreach my $ds (valid_drive_names()) {
2764         next if !defined($conf->{$ds});
2765
2766         my $drive = parse_drive($ds, $conf->{$ds});
2767         next if !$drive;
2768
2769         &$func($ds, $drive, @param);
2770     }
2771 }
2772
2773 sub foreach_volid {
2774     my ($conf, $func, @param) = @_;
2775
2776     my $volhash = {};
2777
2778     my $test_volid = sub {
2779         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2780
2781         return if !$volid;
2782
2783         $volhash->{$volid}->{cdrom} //= 1;
2784         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2785
2786         $volhash->{$volid}->{replicate} //= 0;
2787         $volhash->{$volid}->{replicate} = 1 if $replicate;
2788
2789         $volhash->{$volid}->{referenced_in_config} //= 0;
2790         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2791
2792         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2793             if defined($snapname);
2794     };
2795
2796     foreach_drive($conf, sub {
2797         my ($ds, $drive) = @_;
2798         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2799     });
2800
2801     foreach my $snapname (keys %{$conf->{snapshots}}) {
2802         my $snap = $conf->{snapshots}->{$snapname};
2803         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2804         foreach_drive($snap, sub {
2805             my ($ds, $drive) = @_;
2806             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2807         });
2808     }
2809
2810     foreach my $volid (keys %$volhash) {
2811         &$func($volid, $volhash->{$volid}, @param);
2812     }
2813 }
2814
2815 sub vga_conf_has_spice {
2816     my ($vga) = @_;
2817
2818     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2819
2820     return $1 || 1;
2821 }
2822
2823 sub config_to_command {
2824     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2825
2826     my $cmd = [];
2827     my $globalFlags = [];
2828     my $machineFlags = [];
2829     my $rtcFlags = [];
2830     my $cpuFlags = [];
2831     my $devices = [];
2832     my $pciaddr = '';
2833     my $bridges = {};
2834     my $kvmver = kvm_user_version();
2835     my $vernum = 0; # unknown
2836     my $ostype = $conf->{ostype};
2837     my $winversion = windows_version($ostype);
2838     my $kvm = $conf->{kvm} // 1;
2839
2840     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2841
2842     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2843         $vernum = $1*1000000+$2*1000;
2844     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2845         $vernum = $1*1000000+$2*1000+$3;
2846     }
2847
2848     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2849
2850     my $have_ovz = -f '/proc/vz/vestat';
2851
2852     my $q35 = machine_type_is_q35($conf);
2853     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2854     my $machine_type = $forcemachine || $conf->{machine};
2855     my $use_old_bios_files = undef;
2856     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2857
2858     my $cpuunits = defined($conf->{cpuunits}) ?
2859             $conf->{cpuunits} : $defaults->{cpuunits};
2860
2861     push @$cmd, '/usr/bin/kvm';
2862
2863     push @$cmd, '-id', $vmid;
2864
2865     my $use_virtio = 0;
2866
2867     my $qmpsocket = qmp_socket($vmid);
2868     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2869     push @$cmd, '-mon', "chardev=qmp,mode=control";
2870
2871
2872     push @$cmd, '-pidfile' , pidfile_name($vmid);
2873
2874     push @$cmd, '-daemonize';
2875
2876     if ($conf->{smbios1}) {
2877         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2878     }
2879
2880     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2881         my $ovmfbase;
2882
2883         # prefer the OVMF_CODE variant
2884         if (-f $OVMF_CODE) {
2885             $ovmfbase = $OVMF_CODE;
2886         } elsif (-f $OVMF_IMG) {
2887             $ovmfbase = $OVMF_IMG;
2888         }
2889
2890         die "no uefi base img found\n" if !$ovmfbase;
2891         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2892
2893         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2894             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2895             my $format = $d->{format} // 'raw';
2896             my $path;
2897             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2898             if ($storeid) {
2899                 $path = PVE::Storage::path($storecfg, $d->{file});
2900                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2901                 $format = qemu_img_format($scfg, $volname);
2902             } else {
2903                 $path = $d->{file};
2904                 $format = "raw";
2905             }
2906             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2907         } elsif ($ovmfbase eq $OVMF_CODE) {
2908             warn "using uefi without permanent efivars disk\n";
2909             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2910             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2911             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2912         } else {
2913             # if the base img is not OVMF_CODE, we do not have to bother
2914             # to create/use a vars image, since it will not be used anyway
2915             # this can only happen if someone manually deletes the OVMF_CODE image
2916             # or has an old pve-qemu-kvm version installed.
2917             # both should not happen, but we ignore it here
2918         }
2919     }
2920
2921
2922     # add usb controllers
2923     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2924     push @$devices, @usbcontrollers if @usbcontrollers;
2925     my $vga = $conf->{vga};
2926
2927     my $qxlnum = vga_conf_has_spice($vga);
2928     $vga = 'qxl' if $qxlnum;
2929
2930     if (!$vga) {
2931         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2932             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2933         } else {
2934             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2935         }
2936     }
2937
2938     # enable absolute mouse coordinates (needed by vnc)
2939     my $tablet;
2940     if (defined($conf->{tablet})) {
2941         $tablet = $conf->{tablet};
2942     } else {
2943         $tablet = $defaults->{tablet};
2944         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2945         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2946     }
2947
2948     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2949
2950     my $kvm_off = 0;
2951     my $gpu_passthrough;
2952
2953     # host pci devices
2954     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2955         my $d = parse_hostpci($conf->{"hostpci$i"});
2956         next if !$d;
2957
2958         my $pcie = $d->{pcie};
2959         if($pcie){
2960             die "q35 machine model is not enabled" if !$q35;
2961             $pciaddr = print_pcie_addr("hostpci$i");
2962         }else{
2963             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2964         }
2965
2966         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2967         my $romfile = $d->{romfile};
2968
2969         my $xvga = '';
2970         if ($d->{'x-vga'}) {
2971             $xvga = ',x-vga=on';
2972             $kvm_off = 1;
2973             $vga = 'none';
2974             $gpu_passthrough = 1;
2975
2976             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2977                 $xvga = "";
2978             }
2979         }
2980         my $pcidevices = $d->{pciid};
2981         my $multifunction = 1 if @$pcidevices > 1;
2982
2983         my $j=0;
2984         foreach my $pcidevice (@$pcidevices) {
2985
2986             my $id = "hostpci$i";
2987             $id .= ".$j" if $multifunction;
2988             my $addr = $pciaddr;
2989             $addr .= ".$j" if $multifunction;
2990             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2991
2992             if($j == 0){
2993                 $devicestr .= "$rombar$xvga";
2994                 $devicestr .= ",multifunction=on" if $multifunction;
2995                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
2996             }
2997
2998             push @$devices, '-device', $devicestr;
2999             $j++;
3000         }
3001     }
3002
3003     # usb devices
3004     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3005     push @$devices, @usbdevices if @usbdevices;
3006     # serial devices
3007     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3008         if (my $path = $conf->{"serial$i"}) {
3009             if ($path eq 'socket') {
3010                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3011                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3012                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3013             } else {
3014                 die "no such serial device\n" if ! -c $path;
3015                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3016                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3017             }
3018         }
3019     }
3020
3021     # parallel devices
3022     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3023         if (my $path = $conf->{"parallel$i"}) {
3024             die "no such parallel device\n" if ! -c $path;
3025             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3026             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3027             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3028         }
3029     }
3030
3031     my $vmname = $conf->{name} || "vm$vmid";
3032
3033     push @$cmd, '-name', $vmname;
3034
3035     my $sockets = 1;
3036     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3037     $sockets = $conf->{sockets} if  $conf->{sockets};
3038
3039     my $cores = $conf->{cores} || 1;
3040
3041     my $maxcpus = $sockets * $cores;
3042
3043     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3044
3045     my $allowed_vcpus = $cpuinfo->{cpus};
3046
3047     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3048         if ($allowed_vcpus < $maxcpus);
3049
3050     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3051
3052         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3053         for (my $i = 2; $i <= $vcpus; $i++)  {
3054             my $cpustr = print_cpu_device($conf,$i);
3055             push @$cmd, '-device', $cpustr;
3056         }
3057
3058     } else {
3059
3060         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3061     }
3062     push @$cmd, '-nodefaults';
3063
3064     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3065
3066     my $bootindex_hash = {};
3067     my $i = 1;
3068     foreach my $o (split(//, $bootorder)) {
3069         $bootindex_hash->{$o} = $i*100;
3070         $i++;
3071     }
3072
3073     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3074
3075     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3076
3077     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3078
3079     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3080
3081     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3082         my $socket = vnc_socket($vmid);
3083         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3084     } else {
3085         push @$cmd, '-nographic';
3086     }
3087
3088     # time drift fix
3089     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3090
3091     my $useLocaltime = $conf->{localtime};
3092
3093     if ($winversion >= 5) { # windows
3094         $useLocaltime = 1 if !defined($conf->{localtime});
3095
3096         # use time drift fix when acpi is enabled
3097         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3098             $tdf = 1 if !defined($conf->{tdf});
3099         }
3100     }
3101
3102     if ($winversion >= 6) {
3103         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3104         push @$cmd, '-no-hpet';
3105     }
3106
3107     push @$rtcFlags, 'driftfix=slew' if $tdf;
3108
3109     if (!$kvm) {
3110         push @$machineFlags, 'accel=tcg';
3111     }
3112
3113     if ($machine_type) {
3114         push @$machineFlags, "type=${machine_type}";
3115     }
3116
3117     if ($conf->{startdate}) {
3118         push @$rtcFlags, "base=$conf->{startdate}";
3119     } elsif ($useLocaltime) {
3120         push @$rtcFlags, 'base=localtime';
3121     }
3122
3123     my $cpu = $kvm ? "kvm64" : "qemu64";
3124     if (my $cputype = $conf->{cpu}) {
3125         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3126             or die "Cannot parse cpu description: $cputype\n";
3127         $cpu = $cpuconf->{cputype};
3128         $kvm_off = 1 if $cpuconf->{hidden};
3129     }
3130
3131     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3132
3133     push @$cpuFlags , '-x2apic'
3134         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3135
3136     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3137
3138     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3139
3140     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3141
3142         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3143         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3144     }
3145
3146     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3147
3148     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3149
3150     push @$cpuFlags, 'kvm=off' if $kvm_off;
3151
3152     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3153         die "internal error"; # should not happen
3154
3155     push @$cpuFlags, "vendor=${cpu_vendor}"
3156         if $cpu_vendor ne 'default';
3157
3158     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3159
3160     push @$cmd, '-cpu', $cpu;
3161
3162     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3163
3164     push @$cmd, '-S' if $conf->{freeze};
3165
3166     # set keyboard layout
3167     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3168     push @$cmd, '-k', $kb if $kb;
3169
3170     # enable sound
3171     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3172     #push @$cmd, '-soundhw', 'es1370';
3173     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3174
3175     if($conf->{agent}) {
3176         my $qgasocket = qmp_socket($vmid, 1);
3177         my $pciaddr = print_pci_addr("qga0", $bridges);
3178         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3179         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3180         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3181     }
3182
3183     my $spice_port;
3184
3185     if ($qxlnum) {
3186         if ($qxlnum > 1) {
3187             if ($winversion){
3188                 for(my $i = 1; $i < $qxlnum; $i++){
3189                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3190                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3191                 }
3192             } else {
3193                 # assume other OS works like Linux
3194                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3195                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3196             }
3197         }
3198
3199         my $pciaddr = print_pci_addr("spice", $bridges);
3200
3201         my $nodename = PVE::INotify::nodename();
3202         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3203         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3204         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3205         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3206         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3207
3208         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3209
3210         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3211         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3212         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3213     }
3214
3215     # enable balloon by default, unless explicitly disabled
3216     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3217         $pciaddr = print_pci_addr("balloon0", $bridges);
3218         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3219     }
3220
3221     if ($conf->{watchdog}) {
3222         my $wdopts = parse_watchdog($conf->{watchdog});
3223         $pciaddr = print_pci_addr("watchdog", $bridges);
3224         my $watchdog = $wdopts->{model} || 'i6300esb';
3225         push @$devices, '-device', "$watchdog$pciaddr";
3226         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3227     }
3228
3229     my $vollist = [];
3230     my $scsicontroller = {};
3231     my $ahcicontroller = {};
3232     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3233
3234     # Add iscsi initiator name if available
3235     if (my $initiator = get_initiator_name()) {
3236         push @$devices, '-iscsi', "initiator-name=$initiator";
3237     }
3238
3239     foreach_drive($conf, sub {
3240         my ($ds, $drive) = @_;
3241
3242         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3243             push @$vollist, $drive->{file};
3244         }
3245
3246         $use_virtio = 1 if $ds =~ m/^virtio/;
3247
3248         if (drive_is_cdrom ($drive)) {
3249             if ($bootindex_hash->{d}) {
3250                 $drive->{bootindex} = $bootindex_hash->{d};
3251                 $bootindex_hash->{d} += 1;
3252             }
3253         } else {
3254             if ($bootindex_hash->{c}) {
3255                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3256                 $bootindex_hash->{c} += 1;
3257             }
3258         }
3259
3260         if($drive->{interface} eq 'virtio'){
3261            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3262         }
3263
3264         if ($drive->{interface} eq 'scsi') {
3265
3266             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3267
3268             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3269             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3270
3271             my $iothread = '';
3272             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3273                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3274                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3275             } elsif ($drive->{iothread}) {
3276                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3277             }
3278
3279             my $queues = '';
3280             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3281                 $queues = ",num_queues=$drive->{queues}";
3282             }
3283
3284             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3285             $scsicontroller->{$controller}=1;
3286         }
3287
3288         if ($drive->{interface} eq 'sata') {
3289            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3290            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3291            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3292            $ahcicontroller->{$controller}=1;
3293         }
3294
3295         if ($drive->{interface} eq 'efidisk') {
3296             # this will be added somewhere else
3297             return;
3298         }
3299
3300         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3301         push @$devices, '-drive',$drive_cmd;
3302         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3303     });
3304
3305     for (my $i = 0; $i < $MAX_NETS; $i++) {
3306          next if !$conf->{"net$i"};
3307          my $d = parse_net($conf->{"net$i"});
3308          next if !$d;
3309
3310          $use_virtio = 1 if $d->{model} eq 'virtio';
3311
3312          if ($bootindex_hash->{n}) {
3313             $d->{bootindex} = $bootindex_hash->{n};
3314             $bootindex_hash->{n} += 1;
3315          }
3316
3317          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3318          push @$devices, '-netdev', $netdevfull;
3319
3320          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3321          push @$devices, '-device', $netdevicefull;
3322     }
3323
3324     if (!$q35) {
3325         # add pci bridges
3326         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3327            $bridges->{1} = 1;
3328            $bridges->{2} = 1;
3329         }
3330
3331         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3332
3333         while (my ($k, $v) = each %$bridges) {
3334             $pciaddr = print_pci_addr("pci.$k");
3335             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3336         }
3337     }
3338
3339     # add custom args
3340     if ($conf->{args}) {
3341         my $aa = PVE::Tools::split_args($conf->{args});
3342         push @$cmd, @$aa;
3343     }
3344
3345     push @$cmd, @$devices;
3346     push @$cmd, '-rtc', join(',', @$rtcFlags)
3347         if scalar(@$rtcFlags);
3348     push @$cmd, '-machine', join(',', @$machineFlags)
3349         if scalar(@$machineFlags);
3350     push @$cmd, '-global', join(',', @$globalFlags)
3351         if scalar(@$globalFlags);
3352
3353     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3354 }
3355
3356 sub vnc_socket {
3357     my ($vmid) = @_;
3358     return "${var_run_tmpdir}/$vmid.vnc";
3359 }
3360
3361 sub spice_port {
3362     my ($vmid) = @_;
3363
3364     my $res = vm_mon_cmd($vmid, 'query-spice');
3365
3366     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3367 }
3368
3369 sub qmp_socket {
3370     my ($vmid, $qga) = @_;
3371     my $sockettype = $qga ? 'qga' : 'qmp';
3372     return "${var_run_tmpdir}/$vmid.$sockettype";
3373 }
3374
3375 sub pidfile_name {
3376     my ($vmid) = @_;
3377     return "${var_run_tmpdir}/$vmid.pid";
3378 }
3379
3380 sub vm_devices_list {
3381     my ($vmid) = @_;
3382
3383     my $res = vm_mon_cmd($vmid, 'query-pci');
3384     my $devices = {};
3385     foreach my $pcibus (@$res) {
3386         foreach my $device (@{$pcibus->{devices}}) {
3387             next if !$device->{'qdev_id'};
3388             if ($device->{'pci_bridge'}) {
3389                 $devices->{$device->{'qdev_id'}} = 1;
3390                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3391                     next if !$bridge_device->{'qdev_id'};
3392                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3393                     $devices->{$device->{'qdev_id'}}++;
3394                 }
3395             } else {
3396                 $devices->{$device->{'qdev_id'}} = 1;
3397             }
3398         }
3399     }
3400
3401     my $resblock = vm_mon_cmd($vmid, 'query-block');
3402     foreach my $block (@$resblock) {
3403         if($block->{device} =~ m/^drive-(\S+)/){
3404                 $devices->{$1} = 1;
3405         }
3406     }
3407
3408     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3409     foreach my $mice (@$resmice) {
3410         if ($mice->{name} eq 'QEMU HID Tablet') {
3411             $devices->{tablet} = 1;
3412             last;
3413         }
3414     }
3415
3416     # for usb devices there is no query-usb
3417     # but we can iterate over the entries in
3418     # qom-list path=/machine/peripheral
3419     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3420     foreach my $per (@$resperipheral) {
3421         if ($per->{name} =~ m/^usb\d+$/) {
3422             $devices->{$per->{name}} = 1;
3423         }
3424     }
3425
3426     return $devices;
3427 }
3428
3429 sub vm_deviceplug {
3430     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3431
3432     my $q35 = machine_type_is_q35($conf);
3433
3434     my $devices_list = vm_devices_list($vmid);
3435     return 1 if defined($devices_list->{$deviceid});
3436
3437     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3438
3439     if ($deviceid eq 'tablet') {
3440
3441         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3442
3443     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3444
3445         die "usb hotplug currently not reliable\n";
3446         # since we can't reliably hot unplug all added usb devices
3447         # and usb passthrough disables live migration
3448         # we disable usb hotplugging for now
3449         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3450
3451     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3452
3453         qemu_iothread_add($vmid, $deviceid, $device);
3454
3455         qemu_driveadd($storecfg, $vmid, $device);
3456         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3457
3458         qemu_deviceadd($vmid, $devicefull);
3459         eval { qemu_deviceaddverify($vmid, $deviceid); };
3460         if (my $err = $@) {
3461             eval { qemu_drivedel($vmid, $deviceid); };
3462             warn $@ if $@;
3463             die $err;
3464         }
3465
3466     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3467
3468
3469         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3470         my $pciaddr = print_pci_addr($deviceid);
3471         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3472
3473         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3474
3475         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3476             qemu_iothread_add($vmid, $deviceid, $device);
3477             $devicefull .= ",iothread=iothread-$deviceid";
3478         }
3479
3480         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3481             $devicefull .= ",num_queues=$device->{queues}";
3482         }
3483
3484         qemu_deviceadd($vmid, $devicefull);
3485         qemu_deviceaddverify($vmid, $deviceid);
3486
3487     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3488
3489         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3490         qemu_driveadd($storecfg, $vmid, $device);
3491
3492         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3493         eval { qemu_deviceadd($vmid, $devicefull); };
3494         if (my $err = $@) {
3495             eval { qemu_drivedel($vmid, $deviceid); };
3496             warn $@ if $@;
3497             die $err;
3498         }
3499
3500     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3501
3502         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3503
3504         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3505         my $use_old_bios_files = undef;
3506         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3507
3508         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3509         qemu_deviceadd($vmid, $netdevicefull);
3510         eval { qemu_deviceaddverify($vmid, $deviceid); };
3511         if (my $err = $@) {
3512             eval { qemu_netdevdel($vmid, $deviceid); };
3513             warn $@ if $@;
3514             die $err;
3515         }
3516
3517     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3518
3519         my $bridgeid = $2;
3520         my $pciaddr = print_pci_addr($deviceid);
3521         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3522
3523         qemu_deviceadd($vmid, $devicefull);
3524         qemu_deviceaddverify($vmid, $deviceid);
3525
3526     } else {
3527         die "can't hotplug device '$deviceid'\n";
3528     }
3529
3530     return 1;
3531 }
3532
3533 # fixme: this should raise exceptions on error!
3534 sub vm_deviceunplug {
3535     my ($vmid, $conf, $deviceid) = @_;
3536
3537     my $devices_list = vm_devices_list($vmid);
3538     return 1 if !defined($devices_list->{$deviceid});
3539
3540     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3541
3542     if ($deviceid eq 'tablet') {
3543
3544         qemu_devicedel($vmid, $deviceid);
3545
3546     } elsif ($deviceid =~ m/^usb\d+$/) {
3547
3548         die "usb hotplug currently not reliable\n";
3549         # when unplugging usb devices this way,
3550         # there may be remaining usb controllers/hubs
3551         # so we disable it for now
3552         qemu_devicedel($vmid, $deviceid);
3553         qemu_devicedelverify($vmid, $deviceid);
3554
3555     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3556
3557         qemu_devicedel($vmid, $deviceid);
3558         qemu_devicedelverify($vmid, $deviceid);
3559         qemu_drivedel($vmid, $deviceid);
3560         qemu_iothread_del($conf, $vmid, $deviceid);
3561
3562     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3563
3564         qemu_devicedel($vmid, $deviceid);
3565         qemu_devicedelverify($vmid, $deviceid);
3566         qemu_iothread_del($conf, $vmid, $deviceid);
3567
3568     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3569
3570         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3571         my $device = parse_drive($deviceid, $conf->{$deviceid});
3572         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3573
3574         qemu_devicedel($vmid, $deviceid);
3575         qemu_drivedel($vmid, $deviceid);
3576         qemu_deletescsihw($conf, $vmid, $deviceid);
3577
3578     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3579
3580         qemu_devicedel($vmid, $deviceid);
3581         qemu_devicedelverify($vmid, $deviceid);
3582         qemu_netdevdel($vmid, $deviceid);
3583
3584     } else {
3585         die "can't unplug device '$deviceid'\n";
3586     }
3587
3588     return 1;
3589 }
3590
3591 sub qemu_deviceadd {
3592     my ($vmid, $devicefull) = @_;
3593
3594     $devicefull = "driver=".$devicefull;
3595     my %options =  split(/[=,]/, $devicefull);
3596
3597     vm_mon_cmd($vmid, "device_add" , %options);
3598 }
3599
3600 sub qemu_devicedel {
3601     my ($vmid, $deviceid) = @_;
3602
3603     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3604 }
3605
3606 sub qemu_iothread_add {
3607     my($vmid, $deviceid, $device) = @_;
3608
3609     if ($device->{iothread}) {
3610         my $iothreads = vm_iothreads_list($vmid);
3611         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3612     }
3613 }
3614
3615 sub qemu_iothread_del {
3616     my($conf, $vmid, $deviceid) = @_;
3617
3618     my $device = parse_drive($deviceid, $conf->{$deviceid});
3619     if ($device->{iothread}) {
3620         my $iothreads = vm_iothreads_list($vmid);
3621         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3622     }
3623 }
3624
3625 sub qemu_objectadd {
3626     my($vmid, $objectid, $qomtype) = @_;
3627
3628     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3629
3630     return 1;
3631 }
3632
3633 sub qemu_objectdel {
3634     my($vmid, $objectid) = @_;
3635
3636     vm_mon_cmd($vmid, "object-del", id => $objectid);
3637
3638     return 1;
3639 }
3640
3641 sub qemu_driveadd {
3642     my ($storecfg, $vmid, $device) = @_;
3643
3644     my $drive = print_drive_full($storecfg, $vmid, $device);
3645     $drive =~ s/\\/\\\\/g;
3646     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3647
3648     # If the command succeeds qemu prints: "OK"
3649     return 1 if $ret =~ m/OK/s;
3650
3651     die "adding drive failed: $ret\n";
3652 }
3653
3654 sub qemu_drivedel {
3655     my($vmid, $deviceid) = @_;
3656
3657     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3658     $ret =~ s/^\s+//;
3659
3660     return 1 if $ret eq "";
3661
3662     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3663     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3664
3665     die "deleting drive $deviceid failed : $ret\n";
3666 }
3667
3668 sub qemu_deviceaddverify {
3669     my ($vmid, $deviceid) = @_;
3670
3671     for (my $i = 0; $i <= 5; $i++) {
3672          my $devices_list = vm_devices_list($vmid);
3673          return 1 if defined($devices_list->{$deviceid});
3674          sleep 1;
3675     }
3676
3677     die "error on hotplug device '$deviceid'\n";
3678 }
3679
3680
3681 sub qemu_devicedelverify {
3682     my ($vmid, $deviceid) = @_;
3683
3684     # need to verify that the device is correctly removed as device_del
3685     # is async and empty return is not reliable
3686
3687     for (my $i = 0; $i <= 5; $i++) {
3688          my $devices_list = vm_devices_list($vmid);
3689          return 1 if !defined($devices_list->{$deviceid});
3690          sleep 1;
3691     }
3692
3693     die "error on hot-unplugging device '$deviceid'\n";
3694 }
3695
3696 sub qemu_findorcreatescsihw {
3697     my ($storecfg, $conf, $vmid, $device) = @_;
3698
3699     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3700
3701     my $scsihwid="$controller_prefix$controller";
3702     my $devices_list = vm_devices_list($vmid);
3703
3704     if(!defined($devices_list->{$scsihwid})) {
3705         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3706     }
3707
3708     return 1;
3709 }
3710
3711 sub qemu_deletescsihw {
3712     my ($conf, $vmid, $opt) = @_;
3713
3714     my $device = parse_drive($opt, $conf->{$opt});
3715
3716     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3717         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3718         return 1;
3719     }
3720
3721     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3722
3723     my $devices_list = vm_devices_list($vmid);
3724     foreach my $opt (keys %{$devices_list}) {
3725         if (PVE::QemuServer::is_valid_drivename($opt)) {
3726             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3727             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3728                 return 1;
3729             }
3730         }
3731     }
3732
3733     my $scsihwid="scsihw$controller";
3734
3735     vm_deviceunplug($vmid, $conf, $scsihwid);
3736
3737     return 1;
3738 }
3739
3740 sub qemu_add_pci_bridge {
3741     my ($storecfg, $conf, $vmid, $device) = @_;
3742
3743     my $bridges = {};
3744
3745     my $bridgeid;
3746
3747     print_pci_addr($device, $bridges);
3748
3749     while (my ($k, $v) = each %$bridges) {
3750         $bridgeid = $k;
3751     }
3752     return 1 if !defined($bridgeid) || $bridgeid < 1;
3753
3754     my $bridge = "pci.$bridgeid";
3755     my $devices_list = vm_devices_list($vmid);
3756
3757     if (!defined($devices_list->{$bridge})) {
3758         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3759     }
3760
3761     return 1;
3762 }
3763
3764 sub qemu_set_link_status {
3765     my ($vmid, $device, $up) = @_;
3766
3767     vm_mon_cmd($vmid, "set_link", name => $device,
3768                up => $up ? JSON::true : JSON::false);
3769 }
3770
3771 sub qemu_netdevadd {
3772     my ($vmid, $conf, $device, $deviceid) = @_;
3773
3774     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3775     my %options =  split(/[=,]/, $netdev);
3776
3777     vm_mon_cmd($vmid, "netdev_add",  %options);
3778     return 1;
3779 }
3780
3781 sub qemu_netdevdel {
3782     my ($vmid, $deviceid) = @_;
3783
3784     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3785 }
3786
3787 sub qemu_usb_hotplug {
3788     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3789
3790     return if !$device;
3791
3792     # remove the old one first
3793     vm_deviceunplug($vmid, $conf, $deviceid);
3794
3795     # check if xhci controller is necessary and available
3796     if ($device->{usb3}) {
3797
3798         my $devicelist = vm_devices_list($vmid);
3799
3800         if (!$devicelist->{xhci}) {
3801             my $pciaddr = print_pci_addr("xhci");
3802             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3803         }
3804     }
3805     my $d = parse_usb_device($device->{host});
3806     $d->{usb3} = $device->{usb3};
3807
3808     # add the new one
3809     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3810 }
3811
3812 sub qemu_cpu_hotplug {
3813     my ($vmid, $conf, $vcpus) = @_;
3814
3815     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3816
3817     my $sockets = 1;
3818     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3819     $sockets = $conf->{sockets} if  $conf->{sockets};
3820     my $cores = $conf->{cores} || 1;
3821     my $maxcpus = $sockets * $cores;
3822
3823     $vcpus = $maxcpus if !$vcpus;
3824
3825     die "you can't add more vcpus than maxcpus\n"
3826         if $vcpus > $maxcpus;
3827
3828     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3829
3830     if ($vcpus < $currentvcpus) {
3831
3832         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3833
3834             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3835                 qemu_devicedel($vmid, "cpu$i");
3836                 my $retry = 0;
3837                 my $currentrunningvcpus = undef;
3838                 while (1) {
3839                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3840                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3841                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3842                     $retry++;
3843                     sleep 1;
3844                 }
3845                 #update conf after each succesfull cpu unplug
3846                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3847                 PVE::QemuConfig->write_config($vmid, $conf);
3848             }
3849         } else {
3850             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3851         }
3852
3853         return;
3854     }
3855
3856     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3857     die "vcpus in running vm does not match its configuration\n"
3858         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3859
3860     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3861
3862         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3863             my $cpustr = print_cpu_device($conf, $i);
3864             qemu_deviceadd($vmid, $cpustr);
3865
3866             my $retry = 0;
3867             my $currentrunningvcpus = undef;
3868             while (1) {
3869                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3870                 last if scalar(@{$currentrunningvcpus}) == $i;
3871                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3872                 sleep 1;
3873                 $retry++;
3874             }
3875             #update conf after each succesfull cpu hotplug
3876             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3877             PVE::QemuConfig->write_config($vmid, $conf);
3878         }
3879     } else {
3880
3881         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3882             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3883         }
3884     }
3885 }
3886
3887 sub qemu_block_set_io_throttle {
3888     my ($vmid, $deviceid,
3889         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3890         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3891         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3892         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3893
3894     return if !check_running($vmid) ;
3895
3896     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3897         bps => int($bps),
3898         bps_rd => int($bps_rd),
3899         bps_wr => int($bps_wr),
3900         iops => int($iops),
3901         iops_rd => int($iops_rd),
3902         iops_wr => int($iops_wr),
3903         bps_max => int($bps_max),
3904         bps_rd_max => int($bps_rd_max),
3905         bps_wr_max => int($bps_wr_max),
3906         iops_max => int($iops_max),
3907         iops_rd_max => int($iops_rd_max),
3908         iops_wr_max => int($iops_wr_max),
3909         bps_max_length => int($bps_max_length),
3910         bps_rd_max_length => int($bps_rd_max_length),
3911         bps_wr_max_length => int($bps_wr_max_length),
3912         iops_max_length => int($iops_max_length),
3913         iops_rd_max_length => int($iops_rd_max_length),
3914         iops_wr_max_length => int($iops_wr_max_length),
3915     );
3916
3917 }
3918
3919 # old code, only used to shutdown old VM after update
3920 sub __read_avail {
3921     my ($fh, $timeout) = @_;
3922
3923     my $sel = new IO::Select;
3924     $sel->add($fh);
3925
3926     my $res = '';
3927     my $buf;
3928
3929     my @ready;
3930     while (scalar (@ready = $sel->can_read($timeout))) {
3931         my $count;
3932         if ($count = $fh->sysread($buf, 8192)) {
3933             if ($buf =~ /^(.*)\(qemu\) $/s) {
3934                 $res .= $1;
3935                 last;
3936             } else {
3937                 $res .= $buf;
3938             }
3939         } else {
3940             if (!defined($count)) {
3941                 die "$!\n";
3942             }
3943             last;
3944         }
3945     }
3946
3947     die "monitor read timeout\n" if !scalar(@ready);
3948
3949     return $res;
3950 }
3951
3952 # old code, only used to shutdown old VM after update
3953 sub vm_monitor_command {
3954     my ($vmid, $cmdstr, $nocheck) = @_;
3955
3956     my $res;
3957
3958     eval {
3959         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3960
3961         my $sname = "${var_run_tmpdir}/$vmid.mon";
3962
3963         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3964             die "unable to connect to VM $vmid socket - $!\n";
3965
3966         my $timeout = 3;
3967
3968         # hack: migrate sometime blocks the monitor (when migrate_downtime
3969         # is set)
3970         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3971             $timeout = 60*60; # 1 hour
3972         }
3973
3974         # read banner;
3975         my $data = __read_avail($sock, $timeout);
3976
3977         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3978             die "got unexpected qemu monitor banner\n";
3979         }
3980
3981         my $sel = new IO::Select;
3982         $sel->add($sock);
3983
3984         if (!scalar(my @ready = $sel->can_write($timeout))) {
3985             die "monitor write error - timeout";
3986         }
3987
3988         my $fullcmd = "$cmdstr\r";
3989
3990         # syslog('info', "VM $vmid monitor command: $cmdstr");
3991
3992         my $b;
3993         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3994             die "monitor write error - $!";
3995         }
3996
3997         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3998
3999         $timeout = 20;
4000
4001         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4002             $timeout = 60*60; # 1 hour
4003         } elsif ($cmdstr =~ m/^(eject|change)/) {
4004             $timeout = 60; # note: cdrom mount command is slow
4005         }
4006         if ($res = __read_avail($sock, $timeout)) {
4007
4008             my @lines = split("\r?\n", $res);
4009
4010             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4011
4012             $res = join("\n", @lines);
4013             $res .= "\n";
4014         }
4015     };
4016
4017     my $err = $@;
4018
4019     if ($err) {
4020         syslog("err", "VM $vmid monitor command failed - $err");
4021         die $err;
4022     }
4023
4024     return $res;
4025 }
4026
4027 sub qemu_block_resize {
4028     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4029
4030     my $running = check_running($vmid);
4031
4032     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4033
4034     return if !$running;
4035
4036     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4037
4038 }
4039
4040 sub qemu_volume_snapshot {
4041     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4042
4043     my $running = check_running($vmid);
4044
4045     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4046         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4047     } else {
4048         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4049     }
4050 }
4051
4052 sub qemu_volume_snapshot_delete {
4053     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4054
4055     my $running = check_running($vmid);
4056
4057     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4058         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4059     } else {
4060         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4061     }
4062 }
4063
4064 sub set_migration_caps {
4065     my ($vmid) = @_;
4066
4067     my $cap_ref = [];
4068
4069     my $enabled_cap = {
4070         "auto-converge" => 1,
4071         "xbzrle" => 1,
4072         "x-rdma-pin-all" => 0,
4073         "zero-blocks" => 0,
4074         "compress" => 0
4075     };
4076
4077     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4078
4079     for my $supported_capability (@$supported_capabilities) {
4080         push @$cap_ref, {
4081             capability => $supported_capability->{capability},
4082             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4083         };
4084     }
4085
4086     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4087 }
4088
4089 my $fast_plug_option = {
4090     'lock' => 1,
4091     'name' => 1,
4092     'onboot' => 1,
4093     'shares' => 1,
4094     'startup' => 1,
4095     'description' => 1,
4096     'protection' => 1,
4097 };
4098
4099 # hotplug changes in [PENDING]
4100 # $selection hash can be used to only apply specified options, for
4101 # example: { cores => 1 } (only apply changed 'cores')
4102 # $errors ref is used to return error messages
4103 sub vmconfig_hotplug_pending {
4104     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4105
4106     my $defaults = load_defaults();
4107
4108     # commit values which do not have any impact on running VM first
4109     # Note: those option cannot raise errors, we we do not care about
4110     # $selection and always apply them.
4111
4112     my $add_error = sub {
4113         my ($opt, $msg) = @_;
4114         $errors->{$opt} = "hotplug problem - $msg";
4115     };
4116
4117     my $changes = 0;
4118     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4119         if ($fast_plug_option->{$opt}) {
4120             $conf->{$opt} = $conf->{pending}->{$opt};
4121             delete $conf->{pending}->{$opt};
4122             $changes = 1;
4123         }
4124     }
4125
4126     if ($changes) {
4127         PVE::QemuConfig->write_config($vmid, $conf);
4128         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4129     }
4130
4131     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4132
4133     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4134     while (my ($opt, $force) = each %$pending_delete_hash) {
4135         next if $selection && !$selection->{$opt};
4136         eval {
4137             if ($opt eq 'hotplug') {
4138                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4139             } elsif ($opt eq 'tablet') {
4140                 die "skip\n" if !$hotplug_features->{usb};
4141                 if ($defaults->{tablet}) {
4142                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4143                 } else {
4144                     vm_deviceunplug($vmid, $conf, $opt);
4145                 }
4146             } elsif ($opt =~ m/^usb\d+/) {
4147                 die "skip\n";
4148                 # since we cannot reliably hot unplug usb devices
4149                 # we are disabling it
4150                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4151                 vm_deviceunplug($vmid, $conf, $opt);
4152             } elsif ($opt eq 'vcpus') {
4153                 die "skip\n" if !$hotplug_features->{cpu};
4154                 qemu_cpu_hotplug($vmid, $conf, undef);
4155             } elsif ($opt eq 'balloon') {
4156                 # enable balloon device is not hotpluggable
4157                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4158             } elsif ($fast_plug_option->{$opt}) {
4159                 # do nothing
4160             } elsif ($opt =~ m/^net(\d+)$/) {
4161                 die "skip\n" if !$hotplug_features->{network};
4162                 vm_deviceunplug($vmid, $conf, $opt);
4163             } elsif (is_valid_drivename($opt)) {
4164                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4165                 vm_deviceunplug($vmid, $conf, $opt);
4166                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4167             } elsif ($opt =~ m/^memory$/) {
4168                 die "skip\n" if !$hotplug_features->{memory};
4169                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4170             } elsif ($opt eq 'cpuunits') {
4171                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4172             } elsif ($opt eq 'cpulimit') {
4173                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4174             } else {
4175                 die "skip\n";
4176             }
4177         };
4178         if (my $err = $@) {
4179             &$add_error($opt, $err) if $err ne "skip\n";
4180         } else {
4181             # save new config if hotplug was successful
4182             delete $conf->{$opt};
4183             vmconfig_undelete_pending_option($conf, $opt);
4184             PVE::QemuConfig->write_config($vmid, $conf);
4185             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4186         }
4187     }
4188
4189     foreach my $opt (keys %{$conf->{pending}}) {
4190         next if $selection && !$selection->{$opt};
4191         my $value = $conf->{pending}->{$opt};
4192         eval {
4193             if ($opt eq 'hotplug') {
4194                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4195             } elsif ($opt eq 'tablet') {
4196                 die "skip\n" if !$hotplug_features->{usb};
4197                 if ($value == 1) {
4198                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4199                 } elsif ($value == 0) {
4200                     vm_deviceunplug($vmid, $conf, $opt);
4201                 }
4202             } elsif ($opt =~ m/^usb\d+$/) {
4203                 die "skip\n";
4204                 # since we cannot reliably hot unplug usb devices
4205                 # we are disabling it
4206                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4207                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4208                 die "skip\n" if !$d;
4209                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4210             } elsif ($opt eq 'vcpus') {
4211                 die "skip\n" if !$hotplug_features->{cpu};
4212                 qemu_cpu_hotplug($vmid, $conf, $value);
4213             } elsif ($opt eq 'balloon') {
4214                 # enable/disable balloning device is not hotpluggable
4215                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4216                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4217                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4218
4219                 # allow manual ballooning if shares is set to zero
4220                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4221                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4222                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4223                 }
4224             } elsif ($opt =~ m/^net(\d+)$/) {
4225                 # some changes can be done without hotplug
4226                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4227                                     $vmid, $opt, $value);
4228             } elsif (is_valid_drivename($opt)) {
4229                 # some changes can be done without hotplug
4230                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4231                                      $vmid, $opt, $value, 1);
4232             } elsif ($opt =~ m/^memory$/) { #dimms
4233                 die "skip\n" if !$hotplug_features->{memory};
4234                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4235             } elsif ($opt eq 'cpuunits') {
4236                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4237             } elsif ($opt eq 'cpulimit') {
4238                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4239                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4240             } else {
4241                 die "skip\n";  # skip non-hot-pluggable options
4242             }
4243         };
4244         if (my $err = $@) {
4245             &$add_error($opt, $err) if $err ne "skip\n";
4246         } else {
4247             # save new config if hotplug was successful
4248             $conf->{$opt} = $value;
4249             delete $conf->{pending}->{$opt};
4250             PVE::QemuConfig->write_config($vmid, $conf);
4251             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4252         }
4253     }
4254 }
4255
4256 sub try_deallocate_drive {
4257     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4258
4259     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4260         my $volid = $drive->{file};
4261         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4262             my $sid = PVE::Storage::parse_volume_id($volid);
4263             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4264
4265             # check if the disk is really unused
4266             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4267                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4268             PVE::Storage::vdisk_free($storecfg, $volid);
4269             return 1;
4270         } else {
4271             # If vm is not owner of this disk remove from config
4272             return 1;
4273         }
4274     }
4275
4276     return undef;
4277 }
4278
4279 sub vmconfig_delete_or_detach_drive {
4280     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4281
4282     my $drive = parse_drive($opt, $conf->{$opt});
4283
4284     my $rpcenv = PVE::RPCEnvironment::get();
4285     my $authuser = $rpcenv->get_user();
4286
4287     if ($force) {
4288         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4289         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4290     } else {
4291         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4292     }
4293 }
4294
4295 sub vmconfig_apply_pending {
4296     my ($vmid, $conf, $storecfg) = @_;
4297
4298     # cold plug
4299
4300     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4301     while (my ($opt, $force) = each %$pending_delete_hash) {
4302         die "internal error" if $opt =~ m/^unused/;
4303         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4304         if (!defined($conf->{$opt})) {
4305             vmconfig_undelete_pending_option($conf, $opt);
4306             PVE::QemuConfig->write_config($vmid, $conf);
4307         } elsif (is_valid_drivename($opt)) {
4308             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4309             vmconfig_undelete_pending_option($conf, $opt);
4310             delete $conf->{$opt};
4311             PVE::QemuConfig->write_config($vmid, $conf);
4312         } else {
4313             vmconfig_undelete_pending_option($conf, $opt);
4314             delete $conf->{$opt};
4315             PVE::QemuConfig->write_config($vmid, $conf);
4316         }
4317     }
4318
4319     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4320
4321     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4322         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4323
4324         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4325             # skip if nothing changed
4326         } elsif (is_valid_drivename($opt)) {
4327             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4328                 if defined($conf->{$opt});
4329             $conf->{$opt} = $conf->{pending}->{$opt};
4330         } else {
4331             $conf->{$opt} = $conf->{pending}->{$opt};
4332         }
4333
4334         delete $conf->{pending}->{$opt};
4335         PVE::QemuConfig->write_config($vmid, $conf);
4336     }
4337 }
4338
4339 my $safe_num_ne = sub {
4340     my ($a, $b) = @_;
4341
4342     return 0 if !defined($a) && !defined($b);
4343     return 1 if !defined($a);
4344     return 1 if !defined($b);
4345
4346     return $a != $b;
4347 };
4348
4349 my $safe_string_ne = sub {
4350     my ($a, $b) = @_;
4351
4352     return 0 if !defined($a) && !defined($b);
4353     return 1 if !defined($a);
4354     return 1 if !defined($b);
4355
4356     return $a ne $b;
4357 };
4358
4359 sub vmconfig_update_net {
4360     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4361
4362     my $newnet = parse_net($value);
4363
4364     if ($conf->{$opt}) {
4365         my $oldnet = parse_net($conf->{$opt});
4366
4367         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4368             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4369             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4370             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4371
4372             # for non online change, we try to hot-unplug
4373             die "skip\n" if !$hotplug;
4374             vm_deviceunplug($vmid, $conf, $opt);
4375         } else {
4376
4377             die "internal error" if $opt !~ m/net(\d+)/;
4378             my $iface = "tap${vmid}i$1";
4379
4380             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4381                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4382                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4383                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4384                 PVE::Network::tap_unplug($iface);
4385                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4386             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4387                 # Rate can be applied on its own but any change above needs to
4388                 # include the rate in tap_plug since OVS resets everything.
4389                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4390             }
4391
4392             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4393                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4394             }
4395
4396             return 1;
4397         }
4398     }
4399
4400     if ($hotplug) {
4401         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4402     } else {
4403         die "skip\n";
4404     }
4405 }
4406
4407 sub vmconfig_update_disk {
4408     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4409
4410     # fixme: do we need force?
4411
4412     my $drive = parse_drive($opt, $value);
4413
4414     if ($conf->{$opt}) {
4415
4416         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4417
4418             my $media = $drive->{media} || 'disk';
4419             my $oldmedia = $old_drive->{media} || 'disk';
4420             die "unable to change media type\n" if $media ne $oldmedia;
4421
4422             if (!drive_is_cdrom($old_drive)) {
4423
4424                 if ($drive->{file} ne $old_drive->{file}) {
4425
4426                     die "skip\n" if !$hotplug;
4427
4428                     # unplug and register as unused
4429                     vm_deviceunplug($vmid, $conf, $opt);
4430                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4431
4432                 } else {
4433                     # update existing disk
4434
4435                     # skip non hotpluggable value
4436                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4437                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4438                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4439                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4440                         die "skip\n";
4441                     }
4442
4443                     # apply throttle
4444                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4445                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4446                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4447                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4448                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4449                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4450                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4451                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4452                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4453                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4454                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4455                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4456                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4457                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4458                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4459                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4460                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4461                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4462
4463                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4464                                                    ($drive->{mbps} || 0)*1024*1024,
4465                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4466                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4467                                                    $drive->{iops} || 0,
4468                                                    $drive->{iops_rd} || 0,
4469                                                    $drive->{iops_wr} || 0,
4470                                                    ($drive->{mbps_max} || 0)*1024*1024,
4471                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4472                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4473                                                    $drive->{iops_max} || 0,
4474                                                    $drive->{iops_rd_max} || 0,
4475                                                    $drive->{iops_wr_max} || 0,
4476                                                    $drive->{bps_max_length} || 1,
4477                                                    $drive->{bps_rd_max_length} || 1,
4478                                                    $drive->{bps_wr_max_length} || 1,
4479                                                    $drive->{iops_max_length} || 1,
4480                                                    $drive->{iops_rd_max_length} || 1,
4481                                                    $drive->{iops_wr_max_length} || 1);
4482
4483                     }
4484
4485                     return 1;
4486                 }
4487
4488             } else { # cdrom
4489
4490                 if ($drive->{file} eq 'none') {
4491                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4492                 } else {
4493                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4494                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4495                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4496                 }
4497
4498                 return 1;
4499             }
4500         }
4501     }
4502
4503     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4504     # hotplug new disks
4505     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4506     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4507 }
4508
4509 sub vm_start {
4510     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4511         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4512
4513     PVE::QemuConfig->lock_config($vmid, sub {
4514         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4515
4516         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4517
4518         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4519
4520         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4521
4522         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4523             vmconfig_apply_pending($vmid, $conf, $storecfg);
4524             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4525         }
4526
4527         my $defaults = load_defaults();
4528
4529         # set environment variable useful inside network script
4530         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4531
4532         my $local_volumes = {};
4533
4534         if ($targetstorage) {
4535             foreach_drive($conf, sub {
4536                 my ($ds, $drive) = @_;
4537
4538                 return if drive_is_cdrom($drive);
4539
4540                 my $volid = $drive->{file};
4541
4542                 return if !$volid;
4543
4544                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4545
4546                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4547                 return if $scfg->{shared};
4548                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4549             });
4550
4551             my $format = undef;
4552
4553             foreach my $opt (sort keys %$local_volumes) {
4554
4555                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4556                 my $drive = parse_drive($opt, $conf->{$opt});
4557
4558                 #if remote storage is specified, use default format
4559                 if ($targetstorage && $targetstorage ne "1") {
4560                     $storeid = $targetstorage;
4561                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4562                     $format = $defFormat;
4563                 } else {
4564                     #else we use same format than original
4565                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4566                     $format = qemu_img_format($scfg, $volid);
4567                 }
4568
4569                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4570                 my $newdrive = $drive;
4571                 $newdrive->{format} = $format;
4572                 $newdrive->{file} = $newvolid;
4573                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4574                 $local_volumes->{$opt} = $drivestr;
4575                 #pass drive to conf for command line
4576                 $conf->{$opt} = $drivestr;
4577             }
4578         }
4579
4580         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4581
4582         my $migrate_port = 0;
4583         my $migrate_uri;
4584         if ($statefile) {
4585             if ($statefile eq 'tcp') {
4586                 my $localip = "localhost";
4587                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4588                 my $nodename = PVE::INotify::nodename();
4589
4590                 if (!defined($migration_type)) {
4591                     if (defined($datacenterconf->{migration}->{type})) {
4592                         $migration_type = $datacenterconf->{migration}->{type};
4593                     } else {
4594                         $migration_type = 'secure';
4595                     }
4596                 }
4597
4598                 if ($migration_type eq 'insecure') {
4599                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4600                     if ($migrate_network_addr) {
4601                         $localip = $migrate_network_addr;
4602                     } else {
4603                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4604                     }
4605
4606                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4607                 }
4608
4609                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4610                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4611                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4612                 push @$cmd, '-incoming', $migrate_uri;
4613                 push @$cmd, '-S';
4614
4615             } elsif ($statefile eq 'unix') {
4616                 # should be default for secure migrations as a ssh TCP forward
4617                 # tunnel is not deterministic reliable ready and fails regurarly
4618                 # to set up in time, so use UNIX socket forwards
4619                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4620                 unlink $socket_addr;
4621
4622                 $migrate_uri = "unix:$socket_addr";
4623
4624                 push @$cmd, '-incoming', $migrate_uri;
4625                 push @$cmd, '-S';
4626
4627             } else {
4628                 push @$cmd, '-loadstate', $statefile;
4629             }
4630         } elsif ($paused) {
4631             push @$cmd, '-S';
4632         }
4633
4634         # host pci devices
4635         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4636           my $d = parse_hostpci($conf->{"hostpci$i"});
4637           next if !$d;
4638           my $pcidevices = $d->{pciid};
4639           foreach my $pcidevice (@$pcidevices) {
4640                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4641
4642                 my $info = pci_device_info("0000:$pciid");
4643                 die "IOMMU not present\n" if !check_iommu_support();
4644                 die "no pci device info for device '$pciid'\n" if !$info;
4645                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4646                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4647           }
4648         }
4649
4650         PVE::Storage::activate_volumes($storecfg, $vollist);
4651
4652         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4653             my $cmd = [];
4654             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4655             eval  { run_command($cmd); };
4656         }
4657
4658         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4659                                                   : $defaults->{cpuunits};
4660
4661         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4662
4663         my %properties = (
4664             Slice => 'qemu.slice',
4665             KillMode => 'none',
4666             CPUShares => $cpuunits
4667         );
4668
4669         if (my $cpulimit = $conf->{cpulimit}) {
4670             $properties{CPUQuota} = int($cpulimit * 100);
4671         }
4672         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4673
4674         if ($conf->{hugepages}) {
4675
4676             my $code = sub {
4677                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4678                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4679
4680                 PVE::QemuServer::Memory::hugepages_mount();
4681                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4682
4683                 eval  {
4684                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4685                     run_command($cmd, %run_params);
4686                 };
4687
4688                 if (my $err = $@) {
4689                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4690                     die $err;
4691                 }
4692
4693                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4694             };
4695             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4696
4697         } else {
4698             eval  {
4699                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4700                 run_command($cmd, %run_params);
4701             };
4702         }
4703
4704         if (my $err = $@) {
4705             # deactivate volumes if start fails
4706             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4707             die "start failed: $err";
4708         }
4709
4710         print "migration listens on $migrate_uri\n" if $migrate_uri;
4711
4712         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4713             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4714             warn $@ if $@;
4715         }
4716
4717         #start nbd server for storage migration
4718         if ($targetstorage) {
4719             my $nodename = PVE::INotify::nodename();
4720             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4721             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4722             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4723             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4724
4725             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4726
4727             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4728
4729             foreach my $opt (sort keys %$local_volumes) {
4730                 my $volid = $local_volumes->{$opt};
4731                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4732                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4733                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4734             }
4735         }
4736
4737         if ($migratedfrom) {
4738             eval {
4739                 set_migration_caps($vmid);
4740             };
4741             warn $@ if $@;
4742
4743             if ($spice_port) {
4744                 print "spice listens on port $spice_port\n";
4745                 if ($spice_ticket) {
4746                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4747                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4748                 }
4749             }
4750
4751         } else {
4752             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4753                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4754                     if $conf->{balloon};
4755             }
4756
4757             foreach my $opt (keys %$conf) {
4758                 next if $opt !~  m/^net\d+$/;
4759                 my $nicconf = parse_net($conf->{$opt});
4760                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4761             }
4762         }
4763
4764         vm_mon_cmd_nocheck($vmid, 'qom-set',
4765                     path => "machine/peripheral/balloon0",
4766                     property => "guest-stats-polling-interval",
4767                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4768
4769     });
4770 }
4771
4772 sub vm_mon_cmd {
4773     my ($vmid, $execute, %params) = @_;
4774
4775     my $cmd = { execute => $execute, arguments => \%params };
4776     vm_qmp_command($vmid, $cmd);
4777 }
4778
4779 sub vm_mon_cmd_nocheck {
4780     my ($vmid, $execute, %params) = @_;
4781
4782     my $cmd = { execute => $execute, arguments => \%params };
4783     vm_qmp_command($vmid, $cmd, 1);
4784 }
4785
4786 sub vm_qmp_command {
4787     my ($vmid, $cmd, $nocheck) = @_;
4788
4789     my $res;
4790
4791     my $timeout;
4792     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4793         $timeout = $cmd->{arguments}->{timeout};
4794         delete $cmd->{arguments}->{timeout};
4795     }
4796
4797     eval {
4798         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4799         my $sname = qmp_socket($vmid);
4800         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4801             my $qmpclient = PVE::QMPClient->new();
4802
4803             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4804         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4805             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4806                 if scalar(%{$cmd->{arguments}});
4807             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4808         } else {
4809             die "unable to open monitor socket\n";
4810         }
4811     };
4812     if (my $err = $@) {
4813         syslog("err", "VM $vmid qmp command failed - $err");
4814         die $err;
4815     }
4816
4817     return $res;
4818 }
4819
4820 sub vm_human_monitor_command {
4821     my ($vmid, $cmdline) = @_;
4822
4823     my $res;
4824
4825     my $cmd = {
4826         execute => 'human-monitor-command',
4827         arguments => { 'command-line' => $cmdline},
4828     };
4829
4830     return vm_qmp_command($vmid, $cmd);
4831 }
4832
4833 sub vm_commandline {
4834     my ($storecfg, $vmid) = @_;
4835
4836     my $conf = PVE::QemuConfig->load_config($vmid);
4837
4838     my $defaults = load_defaults();
4839
4840     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4841
4842     return PVE::Tools::cmd2string($cmd);
4843 }
4844
4845 sub vm_reset {
4846     my ($vmid, $skiplock) = @_;
4847
4848     PVE::QemuConfig->lock_config($vmid, sub {
4849
4850         my $conf = PVE::QemuConfig->load_config($vmid);
4851
4852         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4853
4854         vm_mon_cmd($vmid, "system_reset");
4855     });
4856 }
4857
4858 sub get_vm_volumes {
4859     my ($conf) = @_;
4860
4861     my $vollist = [];
4862     foreach_volid($conf, sub {
4863         my ($volid, $attr) = @_;
4864
4865         return if $volid =~ m|^/|;
4866
4867         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4868         return if !$sid;
4869
4870         push @$vollist, $volid;
4871     });
4872
4873     return $vollist;
4874 }
4875
4876 sub vm_stop_cleanup {
4877     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4878
4879     eval {
4880
4881         if (!$keepActive) {
4882             my $vollist = get_vm_volumes($conf);
4883             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4884         }
4885
4886         foreach my $ext (qw(mon qmp pid vnc qga)) {
4887             unlink "/var/run/qemu-server/${vmid}.$ext";
4888         }
4889
4890         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4891     };
4892     warn $@ if $@; # avoid errors - just warn
4893 }
4894
4895 # Note: use $nockeck to skip tests if VM configuration file exists.
4896 # We need that when migration VMs to other nodes (files already moved)
4897 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4898 sub vm_stop {
4899     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4900
4901     $force = 1 if !defined($force) && !$shutdown;
4902
4903     if ($migratedfrom){
4904         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4905         kill 15, $pid if $pid;
4906         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4907         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4908         return;
4909     }
4910
4911     PVE::QemuConfig->lock_config($vmid, sub {
4912
4913         my $pid = check_running($vmid, $nocheck);
4914         return if !$pid;
4915
4916         my $conf;
4917         if (!$nocheck) {
4918             $conf = PVE::QemuConfig->load_config($vmid);
4919             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4920             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4921                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4922                 $timeout = $opts->{down} if $opts->{down};
4923             }
4924         }
4925
4926         $timeout = 60 if !defined($timeout);
4927
4928         eval {
4929             if ($shutdown) {
4930                 if (defined($conf) && $conf->{agent}) {
4931                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4932                 } else {
4933                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4934                 }
4935             } else {
4936                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4937             }
4938         };
4939         my $err = $@;
4940
4941         if (!$err) {
4942             my $count = 0;
4943             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4944                 $count++;
4945                 sleep 1;
4946             }
4947
4948             if ($count >= $timeout) {
4949                 if ($force) {
4950                     warn "VM still running - terminating now with SIGTERM\n";
4951                     kill 15, $pid;
4952                 } else {
4953                     die "VM quit/powerdown failed - got timeout\n";
4954                 }
4955             } else {
4956                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4957                 return;
4958             }
4959         } else {
4960             if ($force) {
4961                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4962                 kill 15, $pid;
4963             } else {
4964                 die "VM quit/powerdown failed\n";
4965             }
4966         }
4967
4968         # wait again
4969         $timeout = 10;
4970
4971         my $count = 0;
4972         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4973             $count++;
4974             sleep 1;
4975         }
4976
4977         if ($count >= $timeout) {
4978             warn "VM still running - terminating now with SIGKILL\n";
4979             kill 9, $pid;
4980             sleep 1;
4981         }
4982
4983         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4984    });
4985 }
4986
4987 sub vm_suspend {
4988     my ($vmid, $skiplock) = @_;
4989
4990     PVE::QemuConfig->lock_config($vmid, sub {
4991
4992         my $conf = PVE::QemuConfig->load_config($vmid);
4993
4994         PVE::QemuConfig->check_lock($conf)
4995             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4996
4997         vm_mon_cmd($vmid, "stop");
4998     });
4999 }
5000
5001 sub vm_resume {
5002     my ($vmid, $skiplock, $nocheck) = @_;
5003
5004     PVE::QemuConfig->lock_config($vmid, sub {
5005
5006         if (!$nocheck) {
5007
5008             my $conf = PVE::QemuConfig->load_config($vmid);
5009
5010             PVE::QemuConfig->check_lock($conf)
5011                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5012
5013             vm_mon_cmd($vmid, "cont");
5014
5015         } else {
5016             vm_mon_cmd_nocheck($vmid, "cont");
5017         }
5018     });
5019 }
5020
5021 sub vm_sendkey {
5022     my ($vmid, $skiplock, $key) = @_;
5023
5024     PVE::QemuConfig->lock_config($vmid, sub {
5025
5026         my $conf = PVE::QemuConfig->load_config($vmid);
5027
5028         # there is no qmp command, so we use the human monitor command
5029         vm_human_monitor_command($vmid, "sendkey $key");
5030     });
5031 }
5032
5033 sub vm_destroy {
5034     my ($storecfg, $vmid, $skiplock) = @_;
5035
5036     PVE::QemuConfig->lock_config($vmid, sub {
5037
5038         my $conf = PVE::QemuConfig->load_config($vmid);
5039
5040         if (!check_running($vmid)) {
5041             destroy_vm($storecfg, $vmid, undef, $skiplock);
5042         } else {
5043             die "VM $vmid is running - destroy failed\n";
5044         }
5045     });
5046 }
5047
5048 # pci helpers
5049
5050 sub file_write {
5051     my ($filename, $buf) = @_;
5052
5053     my $fh = IO::File->new($filename, "w");
5054     return undef if !$fh;
5055
5056     my $res = print $fh $buf;
5057
5058     $fh->close();
5059
5060     return $res;
5061 }
5062
5063 sub pci_device_info {
5064     my ($name) = @_;
5065
5066     my $res;
5067
5068     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5069     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5070
5071     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5072     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5073
5074     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5075     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5076
5077     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5078     return undef if !defined($product) || $product !~ s/^0x//;
5079
5080     $res = {
5081         name => $name,
5082         vendor => $vendor,
5083         product => $product,
5084         domain => $domain,
5085         bus => $bus,
5086         slot => $slot,
5087         func => $func,
5088         irq => $irq,
5089         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5090     };
5091
5092     return $res;
5093 }
5094
5095 sub pci_dev_reset {
5096     my ($dev) = @_;
5097
5098     my $name = $dev->{name};
5099
5100     my $fn = "$pcisysfs/devices/$name/reset";
5101
5102     return file_write($fn, "1");
5103 }
5104
5105 sub pci_dev_bind_to_vfio {
5106     my ($dev) = @_;
5107
5108     my $name = $dev->{name};
5109
5110     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5111
5112     if (!-d $vfio_basedir) {
5113         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5114     }
5115     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5116
5117     my $testdir = "$vfio_basedir/$name";
5118     return 1 if -d $testdir;
5119
5120     my $data = "$dev->{vendor} $dev->{product}";
5121     return undef if !file_write("$vfio_basedir/new_id", $data);
5122
5123     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5124     if (!file_write($fn, $name)) {
5125         return undef if -f $fn;
5126     }
5127
5128     $fn = "$vfio_basedir/bind";
5129     if (! -d $testdir) {
5130         return undef if !file_write($fn, $name);
5131     }
5132
5133     return -d $testdir;
5134 }
5135
5136 sub pci_dev_group_bind_to_vfio {
5137     my ($pciid) = @_;
5138
5139     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5140
5141     if (!-d $vfio_basedir) {
5142         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5143     }
5144     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5145
5146     # get IOMMU group devices
5147     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5148       my @devs = grep /^0000:/, readdir($D);
5149     closedir($D);
5150
5151     foreach my $pciid (@devs) {
5152         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5153
5154         # pci bridges, switches or root ports are not supported
5155         # they have a pci_bus subdirectory so skip them
5156         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5157
5158         my $info = pci_device_info($1);
5159         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5160     }
5161
5162     return 1;
5163 }
5164
5165 # vzdump restore implementaion
5166
5167 sub tar_archive_read_firstfile {
5168     my $archive = shift;
5169
5170     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5171
5172     # try to detect archive type first
5173     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5174         die "unable to open file '$archive'\n";
5175     my $firstfile = <$fh>;
5176     kill 15, $pid;
5177     close $fh;
5178
5179     die "ERROR: archive contaions no data\n" if !$firstfile;
5180     chomp $firstfile;
5181
5182     return $firstfile;
5183 }
5184
5185 sub tar_restore_cleanup {
5186     my ($storecfg, $statfile) = @_;
5187
5188     print STDERR "starting cleanup\n";
5189
5190     if (my $fd = IO::File->new($statfile, "r")) {
5191         while (defined(my $line = <$fd>)) {
5192             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5193                 my $volid = $2;
5194                 eval {
5195                     if ($volid =~ m|^/|) {
5196                         unlink $volid || die 'unlink failed\n';
5197                     } else {
5198                         PVE::Storage::vdisk_free($storecfg, $volid);
5199                     }
5200                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5201                 };
5202                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5203             } else {
5204                 print STDERR "unable to parse line in statfile - $line";
5205             }
5206         }
5207         $fd->close();
5208     }
5209 }
5210
5211 sub restore_archive {
5212     my ($archive, $vmid, $user, $opts) = @_;
5213
5214     my $format = $opts->{format};
5215     my $comp;
5216
5217     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5218         $format = 'tar' if !$format;
5219         $comp = 'gzip';
5220     } elsif ($archive =~ m/\.tar$/) {
5221         $format = 'tar' if !$format;
5222     } elsif ($archive =~ m/.tar.lzo$/) {
5223         $format = 'tar' if !$format;
5224         $comp = 'lzop';
5225     } elsif ($archive =~ m/\.vma$/) {
5226         $format = 'vma' if !$format;
5227     } elsif ($archive =~ m/\.vma\.gz$/) {
5228         $format = 'vma' if !$format;
5229         $comp = 'gzip';
5230     } elsif ($archive =~ m/\.vma\.lzo$/) {
5231         $format = 'vma' if !$format;
5232         $comp = 'lzop';
5233     } else {
5234         $format = 'vma' if !$format; # default
5235     }
5236
5237     # try to detect archive format
5238     if ($format eq 'tar') {
5239         return restore_tar_archive($archive, $vmid, $user, $opts);
5240     } else {
5241         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5242     }
5243 }
5244
5245 sub restore_update_config_line {
5246     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5247
5248     return if $line =~ m/^\#qmdump\#/;
5249     return if $line =~ m/^\#vzdump\#/;
5250     return if $line =~ m/^lock:/;
5251     return if $line =~ m/^unused\d+:/;
5252     return if $line =~ m/^parent:/;
5253     return if $line =~ m/^template:/; # restored VM is never a template
5254
5255     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5256     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5257         # try to convert old 1.X settings
5258         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5259         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5260             my ($model, $macaddr) = split(/\=/, $devconfig);
5261             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5262             my $net = {
5263                 model => $model,
5264                 bridge => "vmbr$ind",
5265                 macaddr => $macaddr,
5266             };
5267             my $netstr = print_net($net);
5268
5269             print $outfd "net$cookie->{netcount}: $netstr\n";
5270             $cookie->{netcount}++;
5271         }
5272     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5273         my ($id, $netstr) = ($1, $2);
5274         my $net = parse_net($netstr);
5275         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5276         $netstr = print_net($net);
5277         print $outfd "$id: $netstr\n";
5278     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5279         my $virtdev = $1;
5280         my $value = $3;
5281         my $di = parse_drive($virtdev, $value);
5282         if (defined($di->{backup}) && !$di->{backup}) {
5283             print $outfd "#$line";
5284         } elsif ($map->{$virtdev}) {
5285             delete $di->{format}; # format can change on restore
5286             $di->{file} = $map->{$virtdev};
5287             $value = print_drive($vmid, $di);
5288             print $outfd "$virtdev: $value\n";
5289         } else {
5290             print $outfd $line;
5291         }
5292     } else {
5293         print $outfd $line;
5294     }
5295 }
5296
5297 sub scan_volids {
5298     my ($cfg, $vmid) = @_;
5299
5300     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5301
5302     my $volid_hash = {};
5303     foreach my $storeid (keys %$info) {
5304         foreach my $item (@{$info->{$storeid}}) {
5305             next if !($item->{volid} && $item->{size});
5306             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5307             $volid_hash->{$item->{volid}} = $item;
5308         }
5309     }
5310
5311     return $volid_hash;
5312 }
5313
5314 sub is_volume_in_use {
5315     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5316
5317     my $path = PVE::Storage::path($storecfg, $volid);
5318
5319     my $scan_config = sub {
5320         my ($cref, $snapname) = @_;
5321
5322         foreach my $key (keys %$cref) {
5323             my $value = $cref->{$key};
5324             if (is_valid_drivename($key)) {
5325                 next if $skip_drive && $key eq $skip_drive;
5326                 my $drive = parse_drive($key, $value);
5327                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5328                 return 1 if $volid eq $drive->{file};
5329                 if ($drive->{file} =~ m!^/!) {
5330                     return 1 if $drive->{file} eq $path;
5331                 } else {
5332                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5333                     next if !$storeid;
5334                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5335                     next if !$scfg;
5336                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5337                 }
5338             }
5339         }
5340
5341         return 0;
5342     };
5343
5344     return 1 if &$scan_config($conf);
5345
5346     undef $skip_drive;
5347
5348     foreach my $snapname (keys %{$conf->{snapshots}}) {
5349         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5350     }
5351
5352     return 0;
5353 }
5354
5355 sub update_disksize {
5356     my ($vmid, $conf, $volid_hash) = @_;
5357
5358     my $changes;
5359
5360     my $used = {};
5361
5362     # Note: it is allowed to define multiple storages with same path (alias), so
5363     # we need to check both 'volid' and real 'path' (two different volid can point
5364     # to the same path).
5365
5366     my $usedpath = {};
5367
5368     # update size info
5369     foreach my $opt (keys %$conf) {
5370         if (is_valid_drivename($opt)) {
5371             my $drive = parse_drive($opt, $conf->{$opt});
5372             my $volid = $drive->{file};
5373             next if !$volid;
5374
5375             $used->{$volid} = 1;
5376             if ($volid_hash->{$volid} &&
5377                 (my $path = $volid_hash->{$volid}->{path})) {
5378                 $usedpath->{$path} = 1;
5379             }
5380
5381             next if drive_is_cdrom($drive);
5382             next if !$volid_hash->{$volid};
5383
5384             $drive->{size} = $volid_hash->{$volid}->{size};
5385             my $new = print_drive($vmid, $drive);
5386             if ($new ne $conf->{$opt}) {
5387                 $changes = 1;
5388                 $conf->{$opt} = $new;
5389             }
5390         }
5391     }
5392
5393     # remove 'unusedX' entry if volume is used
5394     foreach my $opt (keys %$conf) {
5395         next if $opt !~ m/^unused\d+$/;
5396         my $volid = $conf->{$opt};
5397         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5398         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5399             $changes = 1;
5400             delete $conf->{$opt};
5401         }
5402     }
5403
5404     foreach my $volid (sort keys %$volid_hash) {
5405         next if $volid =~ m/vm-$vmid-state-/;
5406         next if $used->{$volid};
5407         my $path = $volid_hash->{$volid}->{path};
5408         next if !$path; # just to be sure
5409         next if $usedpath->{$path};
5410         $changes = 1;
5411         PVE::QemuConfig->add_unused_volume($conf, $volid);
5412         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5413     }
5414
5415     return $changes;
5416 }
5417
5418 sub rescan {
5419     my ($vmid, $nolock) = @_;
5420
5421     my $cfg = PVE::Storage::config();
5422
5423     my $volid_hash = scan_volids($cfg, $vmid);
5424
5425     my $updatefn =  sub {
5426         my ($vmid) = @_;
5427
5428         my $conf = PVE::QemuConfig->load_config($vmid);
5429
5430         PVE::QemuConfig->check_lock($conf);
5431
5432         my $vm_volids = {};
5433         foreach my $volid (keys %$volid_hash) {
5434             my $info = $volid_hash->{$volid};
5435             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5436         }
5437
5438         my $changes = update_disksize($vmid, $conf, $vm_volids);
5439
5440         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5441     };
5442
5443     if (defined($vmid)) {
5444         if ($nolock) {
5445             &$updatefn($vmid);
5446         } else {
5447             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5448         }
5449     } else {
5450         my $vmlist = config_list();
5451         foreach my $vmid (keys %$vmlist) {
5452             if ($nolock) {
5453                 &$updatefn($vmid);
5454             } else {
5455                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5456             }
5457         }
5458     }
5459 }
5460
5461 sub restore_vma_archive {
5462     my ($archive, $vmid, $user, $opts, $comp) = @_;
5463
5464     my $input = $archive eq '-' ? "<&STDIN" : undef;
5465     my $readfrom = $archive;
5466
5467     my $uncomp = '';
5468     if ($comp) {
5469         $readfrom = '-';
5470         my $qarchive = PVE::Tools::shellquote($archive);
5471         if ($comp eq 'gzip') {
5472             $uncomp = "zcat $qarchive|";
5473         } elsif ($comp eq 'lzop') {
5474             $uncomp = "lzop -d -c $qarchive|";
5475         } else {
5476             die "unknown compression method '$comp'\n";
5477         }
5478
5479     }
5480
5481     my $tmpdir = "/var/tmp/vzdumptmp$$";
5482     rmtree $tmpdir;
5483
5484     # disable interrupts (always do cleanups)
5485     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5486         warn "got interrupt - ignored\n";
5487     };
5488
5489     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5490     POSIX::mkfifo($mapfifo, 0600);
5491     my $fifofh;
5492
5493     my $openfifo = sub {
5494         open($fifofh, '>', $mapfifo) || die $!;
5495     };
5496
5497     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5498
5499     my $oldtimeout;
5500     my $timeout = 5;
5501
5502     my $devinfo = {};
5503
5504     my $rpcenv = PVE::RPCEnvironment::get();
5505
5506     my $conffile = PVE::QemuConfig->config_file($vmid);
5507     my $tmpfn = "$conffile.$$.tmp";
5508
5509     # Note: $oldconf is undef if VM does not exists
5510     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5511     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5512
5513     my $print_devmap = sub {
5514         my $virtdev_hash = {};
5515
5516         my $cfgfn = "$tmpdir/qemu-server.conf";
5517
5518         # we can read the config - that is already extracted
5519         my $fh = IO::File->new($cfgfn, "r") ||
5520             "unable to read qemu-server.conf - $!\n";
5521
5522         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5523         if (-f $fwcfgfn) {
5524             my $pve_firewall_dir = '/etc/pve/firewall';
5525             mkdir $pve_firewall_dir; # make sure the dir exists
5526             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5527         }
5528
5529         while (defined(my $line = <$fh>)) {
5530             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5531                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5532                 die "archive does not contain data for drive '$virtdev'\n"
5533                     if !$devinfo->{$devname};
5534                 if (defined($opts->{storage})) {
5535                     $storeid = $opts->{storage} || 'local';
5536                 } elsif (!$storeid) {
5537                     $storeid = 'local';
5538                 }
5539                 $format = 'raw' if !$format;
5540                 $devinfo->{$devname}->{devname} = $devname;
5541                 $devinfo->{$devname}->{virtdev} = $virtdev;
5542                 $devinfo->{$devname}->{format} = $format;
5543                 $devinfo->{$devname}->{storeid} = $storeid;
5544
5545                 # check permission on storage
5546                 my $pool = $opts->{pool}; # todo: do we need that?
5547                 if ($user ne 'root@pam') {
5548                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5549                 }
5550
5551                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5552             }
5553         }
5554
5555         foreach my $devname (keys %$devinfo) {
5556             die "found no device mapping information for device '$devname'\n"
5557                 if !$devinfo->{$devname}->{virtdev};
5558         }
5559
5560         my $cfg = PVE::Storage::config();
5561
5562         # create empty/temp config
5563         if ($oldconf) {
5564             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5565             foreach_drive($oldconf, sub {
5566                 my ($ds, $drive) = @_;
5567
5568                 return if drive_is_cdrom($drive);
5569
5570                 my $volid = $drive->{file};
5571
5572                 return if !$volid || $volid =~ m|^/|;
5573
5574                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5575                 return if !$path || !$owner || ($owner != $vmid);
5576
5577                 # Note: only delete disk we want to restore
5578                 # other volumes will become unused
5579                 if ($virtdev_hash->{$ds}) {
5580                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5581                     if (my $err = $@) {
5582                         warn $err;
5583                     }
5584                 }
5585             });
5586
5587             # delete vmstate files
5588             # since after the restore we have no snapshots anymore
5589             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5590                 my $snap = $oldconf->{snapshots}->{$snapname};
5591                 if ($snap->{vmstate}) {
5592                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5593                     if (my $err = $@) {
5594                         warn $err;
5595                     }
5596                 }
5597             }
5598         }
5599
5600         my $map = {};
5601         foreach my $virtdev (sort keys %$virtdev_hash) {
5602             my $d = $virtdev_hash->{$virtdev};
5603             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5604             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5605
5606             # test if requested format is supported
5607             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5608             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5609             $d->{format} = $defFormat if !$supported;
5610
5611             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5612                                                   $d->{format}, undef, $alloc_size);
5613             print STDERR "new volume ID is '$volid'\n";
5614             $d->{volid} = $volid;
5615             my $path = PVE::Storage::path($cfg, $volid);
5616
5617             PVE::Storage::activate_volumes($cfg,[$volid]);
5618
5619             my $write_zeros = 1;
5620             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5621                 $write_zeros = 0;
5622             }
5623
5624             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5625
5626             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5627             $map->{$virtdev} = $volid;
5628         }
5629
5630         $fh->seek(0, 0) || die "seek failed - $!\n";
5631
5632         my $outfd = new IO::File ($tmpfn, "w") ||
5633             die "unable to write config for VM $vmid\n";
5634
5635         my $cookie = { netcount => 0 };
5636         while (defined(my $line = <$fh>)) {
5637             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5638         }
5639
5640         $fh->close();
5641         $outfd->close();
5642     };
5643
5644     eval {
5645         # enable interrupts
5646         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5647             die "interrupted by signal\n";
5648         };
5649         local $SIG{ALRM} = sub { die "got timeout\n"; };
5650
5651         $oldtimeout = alarm($timeout);
5652
5653         my $parser = sub {
5654             my $line = shift;
5655
5656             print "$line\n";
5657
5658             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5659                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5660                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5661             } elsif ($line =~ m/^CTIME: /) {
5662                 # we correctly received the vma config, so we can disable
5663                 # the timeout now for disk allocation (set to 10 minutes, so
5664                 # that we always timeout if something goes wrong)
5665                 alarm(600);
5666                 &$print_devmap();
5667                 print $fifofh "done\n";
5668                 my $tmp = $oldtimeout || 0;
5669                 $oldtimeout = undef;
5670                 alarm($tmp);
5671                 close($fifofh);
5672             }
5673         };
5674
5675         print "restore vma archive: $cmd\n";
5676         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5677     };
5678     my $err = $@;
5679
5680     alarm($oldtimeout) if $oldtimeout;
5681
5682     my $vollist = [];
5683     foreach my $devname (keys %$devinfo) {
5684         my $volid = $devinfo->{$devname}->{volid};
5685         push @$vollist, $volid if $volid;
5686     }
5687
5688     my $cfg = PVE::Storage::config();
5689     PVE::Storage::deactivate_volumes($cfg, $vollist);
5690
5691     unlink $mapfifo;
5692
5693     if ($err) {
5694         rmtree $tmpdir;
5695         unlink $tmpfn;
5696
5697         foreach my $devname (keys %$devinfo) {
5698             my $volid = $devinfo->{$devname}->{volid};
5699             next if !$volid;
5700             eval {
5701                 if ($volid =~ m|^/|) {
5702                     unlink $volid || die 'unlink failed\n';
5703                 } else {
5704                     PVE::Storage::vdisk_free($cfg, $volid);
5705                 }
5706                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5707             };
5708             print STDERR "unable to cleanup '$volid' - $@" if $@;
5709         }
5710         die $err;
5711     }
5712
5713     rmtree $tmpdir;
5714
5715     rename($tmpfn, $conffile) ||
5716         die "unable to commit configuration file '$conffile'\n";
5717
5718     PVE::Cluster::cfs_update(); # make sure we read new file
5719
5720     eval { rescan($vmid, 1); };
5721     warn $@ if $@;
5722 }
5723
5724 sub restore_tar_archive {
5725     my ($archive, $vmid, $user, $opts) = @_;
5726
5727     if ($archive ne '-') {
5728         my $firstfile = tar_archive_read_firstfile($archive);
5729         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5730             if $firstfile ne 'qemu-server.conf';
5731     }
5732
5733     my $storecfg = PVE::Storage::config();
5734
5735     # destroy existing data - keep empty config
5736     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5737     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5738
5739     my $tocmd = "/usr/lib/qemu-server/qmextract";
5740
5741     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5742     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5743     $tocmd .= ' --prealloc' if $opts->{prealloc};
5744     $tocmd .= ' --info' if $opts->{info};
5745
5746     # tar option "xf" does not autodetect compression when read from STDIN,
5747     # so we pipe to zcat
5748     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5749         PVE::Tools::shellquote("--to-command=$tocmd");
5750
5751     my $tmpdir = "/var/tmp/vzdumptmp$$";
5752     mkpath $tmpdir;
5753
5754     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5755     local $ENV{VZDUMP_VMID} = $vmid;
5756     local $ENV{VZDUMP_USER} = $user;
5757
5758     my $conffile = PVE::QemuConfig->config_file($vmid);
5759     my $tmpfn = "$conffile.$$.tmp";
5760
5761     # disable interrupts (always do cleanups)
5762     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5763         print STDERR "got interrupt - ignored\n";
5764     };
5765
5766     eval {
5767         # enable interrupts
5768         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5769             die "interrupted by signal\n";
5770         };
5771
5772         if ($archive eq '-') {
5773             print "extracting archive from STDIN\n";
5774             run_command($cmd, input => "<&STDIN");
5775         } else {
5776             print "extracting archive '$archive'\n";
5777             run_command($cmd);
5778         }
5779
5780         return if $opts->{info};
5781
5782         # read new mapping
5783         my $map = {};
5784         my $statfile = "$tmpdir/qmrestore.stat";
5785         if (my $fd = IO::File->new($statfile, "r")) {
5786             while (defined (my $line = <$fd>)) {
5787                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5788                     $map->{$1} = $2 if $1;
5789                 } else {
5790                     print STDERR "unable to parse line in statfile - $line\n";
5791                 }
5792             }
5793             $fd->close();
5794         }
5795
5796         my $confsrc = "$tmpdir/qemu-server.conf";
5797
5798         my $srcfd = new IO::File($confsrc, "r") ||
5799             die "unable to open file '$confsrc'\n";
5800
5801         my $outfd = new IO::File ($tmpfn, "w") ||
5802             die "unable to write config for VM $vmid\n";
5803
5804         my $cookie = { netcount => 0 };
5805         while (defined (my $line = <$srcfd>)) {
5806             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5807         }
5808
5809         $srcfd->close();
5810         $outfd->close();
5811     };
5812     my $err = $@;
5813
5814     if ($err) {
5815
5816         unlink $tmpfn;
5817
5818         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5819
5820         die $err;
5821     }
5822
5823     rmtree $tmpdir;
5824
5825     rename $tmpfn, $conffile ||
5826         die "unable to commit configuration file '$conffile'\n";
5827
5828     PVE::Cluster::cfs_update(); # make sure we read new file
5829
5830     eval { rescan($vmid, 1); };
5831     warn $@ if $@;
5832 };
5833
5834 sub foreach_writable_storage {
5835     my ($conf, $func) = @_;
5836
5837     my $sidhash = {};
5838
5839     foreach my $ds (keys %$conf) {
5840         next if !is_valid_drivename($ds);
5841
5842         my $drive = parse_drive($ds, $conf->{$ds});
5843         next if !$drive;
5844         next if drive_is_cdrom($drive);
5845
5846         my $volid = $drive->{file};
5847
5848         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5849         $sidhash->{$sid} = $sid if $sid;
5850     }
5851
5852     foreach my $sid (sort keys %$sidhash) {
5853         &$func($sid);
5854     }
5855 }
5856
5857 sub do_snapshots_with_qemu {
5858     my ($storecfg, $volid) = @_;
5859
5860     my $storage_name = PVE::Storage::parse_volume_id($volid);
5861
5862     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5863         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5864         return 1;
5865     }
5866
5867     if ($volid =~ m/\.(qcow2|qed)$/){
5868         return 1;
5869     }
5870
5871     return undef;
5872 }
5873
5874 sub qga_check_running {
5875     my ($vmid) = @_;
5876
5877     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5878     if ($@) {
5879         warn "Qemu Guest Agent are not running - $@";
5880         return 0;
5881     }
5882     return 1;
5883 }
5884
5885 sub template_create {
5886     my ($vmid, $conf, $disk) = @_;
5887
5888     my $storecfg = PVE::Storage::config();
5889
5890     foreach_drive($conf, sub {
5891         my ($ds, $drive) = @_;
5892
5893         return if drive_is_cdrom($drive);
5894         return if $disk && $ds ne $disk;
5895
5896         my $volid = $drive->{file};
5897         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5898
5899         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5900         $drive->{file} = $voliddst;
5901         $conf->{$ds} = print_drive($vmid, $drive);
5902         PVE::QemuConfig->write_config($vmid, $conf);
5903     });
5904 }
5905
5906 sub qemu_img_convert {
5907     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5908
5909     my $storecfg = PVE::Storage::config();
5910     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5911     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5912
5913     if ($src_storeid && $dst_storeid) {
5914
5915         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5916
5917         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5918         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5919
5920         my $src_format = qemu_img_format($src_scfg, $src_volname);
5921         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5922
5923         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5924         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5925
5926         my $cmd = [];
5927         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5928         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5929         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5930         if ($is_zero_initialized) {
5931             push @$cmd, "zeroinit:$dst_path";
5932         } else {
5933             push @$cmd, $dst_path;
5934         }
5935
5936         my $parser = sub {
5937             my $line = shift;
5938             if($line =~ m/\((\S+)\/100\%\)/){
5939                 my $percent = $1;
5940                 my $transferred = int($size * $percent / 100);
5941                 my $remaining = $size - $transferred;
5942
5943                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5944             }
5945
5946         };
5947
5948         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5949         my $err = $@;
5950         die "copy failed: $err" if $err;
5951     }
5952 }
5953
5954 sub qemu_img_format {
5955     my ($scfg, $volname) = @_;
5956
5957     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5958         return $1;
5959     } else {
5960         return "raw";
5961     }
5962 }
5963
5964 sub qemu_drive_mirror {
5965     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5966
5967     $jobs = {} if !$jobs;
5968
5969     my $qemu_target;
5970     my $format;
5971     $jobs->{"drive-$drive"} = {};
5972
5973     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5974         my $server = $1;
5975         my $port = $2;
5976         my $exportname = $3;
5977
5978         $format = "nbd";
5979         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5980         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5981         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5982
5983         my $pid = fork();
5984         if (!defined($pid)) {
5985             die "forking socat tunnel failed\n";
5986         } elsif ($pid == 0) {
5987             exec(@$cmd);
5988             warn "exec failed: $!\n";
5989             POSIX::_exit(-1);
5990         }
5991         $jobs->{"drive-$drive"}->{pid} = $pid;
5992
5993         my $timeout = 0;
5994         while (!-S $unixsocket) {
5995             die "nbd connection helper timed out\n"
5996                 if $timeout++ > 5;
5997             sleep 1;
5998         }
5999     } else {
6000         my $storecfg = PVE::Storage::config();
6001         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6002
6003         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6004
6005         $format = qemu_img_format($dst_scfg, $dst_volname);
6006
6007         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6008
6009         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6010     }
6011
6012     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6013     $opts->{format} = $format if $format;
6014
6015     print "drive mirror is starting for drive-$drive\n";
6016
6017     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6018
6019     if (my $err = $@) {
6020         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6021         die "mirroring error: $err";
6022     }
6023
6024     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6025 }
6026
6027 sub qemu_drive_mirror_monitor {
6028     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6029
6030     eval {
6031         my $err_complete = 0;
6032
6033         while (1) {
6034             die "storage migration timed out\n" if $err_complete > 300;
6035
6036             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6037
6038             my $running_mirror_jobs = {};
6039             foreach my $stat (@$stats) {
6040                 next if $stat->{type} ne 'mirror';
6041                 $running_mirror_jobs->{$stat->{device}} = $stat;
6042             }
6043
6044             my $readycounter = 0;
6045
6046             foreach my $job (keys %$jobs) {
6047
6048                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6049                     print "$job : finished\n";
6050                     delete $jobs->{$job};
6051                     next;
6052                 }
6053
6054                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6055
6056                 my $busy = $running_mirror_jobs->{$job}->{busy};
6057                 my $ready = $running_mirror_jobs->{$job}->{ready};
6058                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6059                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6060                     my $remaining = $total - $transferred;
6061                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6062
6063                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6064                 }
6065
6066                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6067             }
6068
6069             last if scalar(keys %$jobs) == 0;
6070
6071             if ($readycounter == scalar(keys %$jobs)) {
6072                 print "all mirroring jobs are ready \n";
6073                 last if $skipcomplete; #do the complete later
6074
6075                 if ($vmiddst && $vmiddst != $vmid) {
6076                     if ($qga) {
6077                         print "freeze filesystem\n";
6078                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6079                     } else {
6080                         print "suspend vm\n";
6081                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6082                     }
6083
6084                     # if we clone a disk for a new target vm, we don't switch the disk
6085                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6086
6087                     if ($qga) {
6088                         print "unfreeze filesystem\n";
6089                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6090                     } else {
6091                         print "resume vm\n";
6092                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6093                     }
6094
6095                     last;
6096                 } else {
6097
6098                     foreach my $job (keys %$jobs) {
6099                         # try to switch the disk if source and destination are on the same guest
6100                         print "$job: Completing block job...\n";
6101
6102                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6103                         if ($@ =~ m/cannot be completed/) {
6104                             print "$job: Block job cannot be completed, try again.\n";
6105                             $err_complete++;
6106                         }else {
6107                             print "$job: Completed successfully.\n";
6108                             $jobs->{$job}->{complete} = 1;
6109                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6110                         }
6111                     }
6112                 }
6113             }
6114             sleep 1;
6115         }
6116     };
6117     my $err = $@;
6118
6119     if ($err) {
6120         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6121         die "mirroring error: $err";
6122     }
6123
6124 }
6125
6126 sub qemu_blockjobs_cancel {
6127     my ($vmid, $jobs) = @_;
6128
6129     foreach my $job (keys %$jobs) {
6130         print "$job: Cancelling block job\n";
6131         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6132         $jobs->{$job}->{cancel} = 1;
6133     }
6134
6135     while (1) {
6136         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6137
6138         my $running_jobs = {};
6139         foreach my $stat (@$stats) {
6140             $running_jobs->{$stat->{device}} = $stat;
6141         }
6142
6143         foreach my $job (keys %$jobs) {
6144
6145             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6146                 print "$job: Done.\n";
6147                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6148                 delete $jobs->{$job};
6149             }
6150         }
6151
6152         last if scalar(keys %$jobs) == 0;
6153
6154         sleep 1;
6155     }
6156 }
6157
6158 sub qemu_blockjobs_finish_tunnel {
6159    my ($vmid, $job, $cpid) = @_;
6160
6161    return if !$cpid;
6162
6163    for (my $i = 1; $i < 20; $i++) {
6164         my $waitpid = waitpid($cpid, WNOHANG);
6165         last if (defined($waitpid) && ($waitpid == $cpid));
6166
6167         if ($i == 10) {
6168             kill(15, $cpid);
6169         } elsif ($i >= 15) {
6170             kill(9, $cpid);
6171         }
6172         sleep (1);
6173     }
6174     unlink "/run/qemu-server/$vmid.mirror-$job";
6175 }
6176
6177 sub clone_disk {
6178     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6179         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6180
6181     my $newvolid;
6182
6183     if (!$full) {
6184         print "create linked clone of drive $drivename ($drive->{file})\n";
6185         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6186         push @$newvollist, $newvolid;
6187     } else {
6188
6189         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6190         $storeid = $storage if $storage;
6191
6192         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6193         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6194
6195         print "create full clone of drive $drivename ($drive->{file})\n";
6196         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6197         push @$newvollist, $newvolid;
6198
6199         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6200
6201         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6202         if (!$running || $snapname) {
6203             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6204         } else {
6205
6206             my $kvmver = get_running_qemu_version ($vmid);
6207             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6208                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6209                     if $drive->{iothread};
6210             }
6211
6212             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6213         }
6214     }
6215
6216     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6217
6218     my $disk = $drive;
6219     $disk->{format} = undef;
6220     $disk->{file} = $newvolid;
6221     $disk->{size} = $size;
6222
6223     return $disk;
6224 }
6225
6226 # this only works if VM is running
6227 sub get_current_qemu_machine {
6228     my ($vmid) = @_;
6229
6230     my $cmd = { execute => 'query-machines', arguments => {} };
6231     my $res = vm_qmp_command($vmid, $cmd);
6232
6233     my ($current, $default);
6234     foreach my $e (@$res) {
6235         $default = $e->{name} if $e->{'is-default'};
6236         $current = $e->{name} if $e->{'is-current'};
6237     }
6238
6239     # fallback to the default machine if current is not supported by qemu
6240     return $current || $default || 'pc';
6241 }
6242
6243 sub get_running_qemu_version {
6244     my ($vmid) = @_;
6245     my $cmd = { execute => 'query-version', arguments => {} };
6246     my $res = vm_qmp_command($vmid, $cmd);
6247     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6248 }
6249
6250 sub qemu_machine_feature_enabled {
6251     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6252
6253     my $current_major;
6254     my $current_minor;
6255
6256     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6257
6258         $current_major = $3;
6259         $current_minor = $4;
6260
6261     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6262
6263         $current_major = $1;
6264         $current_minor = $2;
6265     }
6266
6267     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6268
6269
6270 }
6271
6272 sub qemu_machine_pxe {
6273     my ($vmid, $conf, $machine) = @_;
6274
6275     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6276
6277     foreach my $opt (keys %$conf) {
6278         next if $opt !~ m/^net(\d+)$/;
6279         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6280         next if !$net;
6281         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6282         return $machine.".pxe" if $romfile =~ m/pxe/;
6283         last;
6284     }
6285
6286     return $machine;
6287 }
6288
6289 sub qemu_use_old_bios_files {
6290     my ($machine_type) = @_;
6291
6292     return if !$machine_type;
6293
6294     my $use_old_bios_files = undef;
6295
6296     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6297         $machine_type = $1;
6298         $use_old_bios_files = 1;
6299     } else {
6300         my $kvmver = kvm_user_version();
6301         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6302         # load new efi bios files on migration. So this hack is required to allow
6303         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6304         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6305         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6306     }
6307
6308     return ($use_old_bios_files, $machine_type);
6309 }
6310
6311 sub lspci {
6312
6313     my $devices = {};
6314
6315     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6316             my (undef, $id, $function) = @_;
6317             my $res = { id => $id, function => $function};
6318             push @{$devices->{$id}}, $res;
6319     });
6320
6321     # Entries should be sorted by functions.
6322     foreach my $id (keys %$devices) {
6323         my $dev = $devices->{$id};
6324         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6325     }
6326
6327     return $devices;
6328 }
6329
6330 sub vm_iothreads_list {
6331     my ($vmid) = @_;
6332
6333     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6334
6335     my $iothreads = {};
6336     foreach my $iothread (@$res) {
6337         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6338     }
6339
6340     return $iothreads;
6341 }
6342
6343 sub scsihw_infos {
6344     my ($conf, $drive) = @_;
6345
6346     my $maxdev = 0;
6347
6348     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6349         $maxdev = 7;
6350     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6351         $maxdev = 1;
6352     } else {
6353         $maxdev = 256;
6354     }
6355
6356     my $controller = int($drive->{index} / $maxdev);
6357     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6358
6359     return ($maxdev, $controller, $controller_prefix);
6360 }
6361
6362 sub add_hyperv_enlightenments {
6363     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6364
6365     return if $winversion < 6;
6366     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6367
6368     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6369
6370     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6371         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6372         push @$cpuFlags , 'hv_vapic';
6373         push @$cpuFlags , 'hv_time';
6374     } else {
6375         push @$cpuFlags , 'hv_spinlocks=0xffff';
6376     }
6377
6378     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6379         push @$cpuFlags , 'hv_reset';
6380         push @$cpuFlags , 'hv_vpindex';
6381         push @$cpuFlags , 'hv_runtime';
6382     }
6383
6384     if ($winversion >= 7) {
6385         push @$cpuFlags , 'hv_relaxed';
6386     }
6387 }
6388
6389 sub windows_version {
6390     my ($ostype) = @_;
6391
6392     return 0 if !$ostype;
6393
6394     my $winversion = 0;
6395
6396     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6397         $winversion = 5;
6398     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6399         $winversion = 6;
6400     } elsif ($ostype =~ m/^win(\d+)$/) {
6401         $winversion = $1;
6402     }
6403
6404     return $winversion;
6405 }
6406
6407 sub resolve_dst_disk_format {
6408         my ($storecfg, $storeid, $src_volname, $format) = @_;
6409         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6410
6411         if (!$format) {
6412             # if no target format is specified, use the source disk format as hint
6413             if ($src_volname) {
6414                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6415                 $format = qemu_img_format($scfg, $src_volname);
6416             } else {
6417                 return $defFormat;
6418             }
6419         }
6420
6421         # test if requested format is supported - else use default
6422         my $supported = grep { $_ eq $format } @$validFormats;
6423         $format = $defFormat if !$supported;
6424         return $format;
6425 }
6426
6427 # bash completion helper
6428
6429 sub complete_backup_archives {
6430     my ($cmdname, $pname, $cvalue) = @_;
6431
6432     my $cfg = PVE::Storage::config();
6433
6434     my $storeid;
6435
6436     if ($cvalue =~ m/^([^:]+):/) {
6437         $storeid = $1;
6438     }
6439
6440     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6441
6442     my $res = [];
6443     foreach my $id (keys %$data) {
6444         foreach my $item (@{$data->{$id}}) {
6445             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6446             push @$res, $item->{volid} if defined($item->{volid});
6447         }
6448     }
6449
6450     return $res;
6451 }
6452
6453 my $complete_vmid_full = sub {
6454     my ($running) = @_;
6455
6456     my $idlist = vmstatus();
6457
6458     my $res = [];
6459
6460     foreach my $id (keys %$idlist) {
6461         my $d = $idlist->{$id};
6462         if (defined($running)) {
6463             next if $d->{template};
6464             next if $running && $d->{status} ne 'running';
6465             next if !$running && $d->{status} eq 'running';
6466         }
6467         push @$res, $id;
6468
6469     }
6470     return $res;
6471 };
6472
6473 sub complete_vmid {
6474     return &$complete_vmid_full();
6475 }
6476
6477 sub complete_vmid_stopped {
6478     return &$complete_vmid_full(0);
6479 }
6480
6481 sub complete_vmid_running {
6482     return &$complete_vmid_full(1);
6483 }
6484
6485 sub complete_storage {
6486
6487     my $cfg = PVE::Storage::config();
6488     my $ids = $cfg->{ids};
6489
6490     my $res = [];
6491     foreach my $sid (keys %$ids) {
6492         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6493         next if !$ids->{$sid}->{content}->{images};
6494         push @$res, $sid;
6495     }
6496
6497     return $res;
6498 }
6499
6500 sub nbd_stop {
6501     my ($vmid) = @_;
6502
6503     vm_mon_cmd($vmid, 'nbd-server-stop');
6504 }
6505
6506 1;