PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  60     description => "Some command save/restore state from this location.",
  61     type => 'string',
  62     maxLength => 128,
  63     optional => 1,
  64 });
  65
  66 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  67     description => "The name of the snapshot.",
  68     type => 'string', format => 'pve-configid',
  69     maxLength => 40,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  73     type => 'string',
  74     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  75     description => "The drive's backing file's data format.",
  76     optional => 1,
  77 });
  78
  79 #no warnings 'redefine';
  80
  81 sub cgroups_write {
  82    my ($controller, $vmid, $option, $value) = @_;
  83
  84    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  85    PVE::ProcFSTools::write_proc_entry($path, $value);
  86
  87 }
  88
  89 my $nodename = PVE::INotify::nodename();
  90
  91 mkdir "/etc/pve/nodes/$nodename";
  92 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  93 mkdir $confdir;
  94
  95 my $var_run_tmpdir = "/var/run/qemu-server";
  96 mkdir $var_run_tmpdir;
  97
  98 my $lock_dir = "/var/lock/qemu-server";
  99 mkdir $lock_dir;
 100
 101 my $pcisysfs = "/sys/bus/pci";
 102
 103 my $cpu_vendor_list = {
 104     # Intel CPUs
 105     486 => 'GenuineIntel',
 106     pentium => 'GenuineIntel',
 107     pentium2  => 'GenuineIntel',
 108     pentium3  => 'GenuineIntel',
 109     coreduo => 'GenuineIntel',
 110     core2duo => 'GenuineIntel',
 111     Conroe  => 'GenuineIntel',
 112     Penryn  => 'GenuineIntel',
 113     Nehalem  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     SandyBridge => 'GenuineIntel',
 116     IvyBridge => 'GenuineIntel',
 117     Haswell => 'GenuineIntel',
 118     'Haswell-noTSX' => 'GenuineIntel',
 119     Broadwell => 'GenuineIntel',
 120     'Broadwell-noTSX' => 'GenuineIntel',
 121     'Skylake-Client' => 'GenuineIntel',
 122
 123     # AMD CPUs
 124     athlon => 'AuthenticAMD',
 125     phenom  => 'AuthenticAMD',
 126     Opteron_G1  => 'AuthenticAMD',
 127     Opteron_G2  => 'AuthenticAMD',
 128     Opteron_G3  => 'AuthenticAMD',
 129     Opteron_G4  => 'AuthenticAMD',
 130     Opteron_G5  => 'AuthenticAMD',
 131
 132     # generic types, use vendor from host node
 133     host => 'default',
 134     kvm32 => 'default',
 135     kvm64 => 'default',
 136     qemu32 => 'default',
 137     qemu64 => 'default',
 138 };
 139
 140 my $cpu_fmt = {
 141     cputype => {
 142         description => "Emulated CPU type.",
 143         type => 'string',
 144         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 145         default => 'kvm64',
 146         default_key => 1,
 147     },
 148     hidden => {
 149         description => "Do not identify as a KVM virtual machine.",
 150         type => 'boolean',
 151         optional => 1,
 152         default => 0
 153     },
 154 };
 155
 156 my $watchdog_fmt = {
 157     model => {
 158         default_key => 1,
 159         type => 'string',
 160         enum => [qw(i6300esb ib700)],
 161         description => "Watchdog type to emulate.",
 162         default => 'i6300esb',
 163         optional => 1,
 164     },
 165     action => {
 166         type => 'string',
 167         enum => [qw(reset shutdown poweroff pause debug none)],
 168         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 169         optional => 1,
 170     },
 171 };
 172 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 173
 174 my $confdesc = {
 175     onboot => {
 176         optional => 1,
 177         type => 'boolean',
 178         description => "Specifies whether a VM will be started during system bootup.",
 179         default => 0,
 180     },
 181     autostart => {
 182         optional => 1,
 183         type => 'boolean',
 184         description => "Automatic restart after crash (currently ignored).",
 185         default => 0,
 186     },
 187     hotplug => {
 188         optional => 1,
 189         type => 'string', format => 'pve-hotplug-features',
 190         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 191         default => 'network,disk,usb',
 192     },
 193     reboot => {
 194         optional => 1,
 195         type => 'boolean',
 196         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 197         default => 1,
 198     },
 199     lock => {
 200         optional => 1,
 201         type => 'string',
 202         description => "Lock/unlock the VM.",
 203         enum => [qw(migrate backup snapshot rollback)],
 204     },
 205     cpulimit => {
 206         optional => 1,
 207         type => 'number',
 208         description => "Limit of CPU usage.",
 209         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 210         minimum => 0,
 211         maximum => 128,
 212         default => 0,
 213     },
 214     cpuunits => {
 215         optional => 1,
 216         type => 'integer',
 217         description => "CPU weight for a VM.",
 218         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 219         minimum => 0,
 220         maximum => 500000,
 221         default => 1024,
 222     },
 223     memory => {
 224         optional => 1,
 225         type => 'integer',
 226         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 227         minimum => 16,
 228         default => 512,
 229     },
 230     balloon => {
 231         optional => 1,
 232         type => 'integer',
 233         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 234         minimum => 0,
 235     },
 236     shares => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 240         minimum => 0,
 241         maximum => 50000,
 242         default => 1000,
 243     },
 244     keyboard => {
 245         optional => 1,
 246         type => 'string',
 247         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 248         enum => PVE::Tools::kvmkeymaplist(),
 249         default => 'en-us',
 250     },
 251     name => {
 252         optional => 1,
 253         type => 'string', format => 'dns-name',
 254         description => "Set a name for the VM. Only used on the configuration web interface.",
 255     },
 256     scsihw => {
 257         optional => 1,
 258         type => 'string',
 259         description => "SCSI controller model",
 260         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 261         default => 'lsi',
 262     },
 263     description => {
 264         optional => 1,
 265         type => 'string',
 266         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 267     },
 268     ostype => {
 269         optional => 1,
 270         type => 'string',
 271         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 272         description => "Specify guest operating system.",
 273         verbose_description => <<EODESC,
 274 Specify guest operating system. This is used to enable special
 275 optimization/features for specific operating systems:
 276
 277 [horizontal]
 278 other;; unspecified OS
 279 wxp;; Microsoft Windows XP
 280 w2k;; Microsoft Windows 2000
 281 w2k3;; Microsoft Windows 2003
 282 w2k8;; Microsoft Windows 2008
 283 wvista;; Microsoft Windows Vista
 284 win7;; Microsoft Windows 7
 285 win8;; Microsoft Windows 8/2012
 286 l24;; Linux 2.4 Kernel
 287 l26;; Linux 2.6/3.X Kernel
 288 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 289 EODESC
 290     },
 291     boot => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 295         pattern => '[acdn]{1,4}',
 296         default => 'cdn',
 297     },
 298     bootdisk => {
 299         optional => 1,
 300         type => 'string', format => 'pve-qm-bootdisk',
 301         description => "Enable booting from specified disk.",
 302         pattern => '(ide|sata|scsi|virtio)\d+',
 303     },
 304     smp => {
 305         optional => 1,
 306         type => 'integer',
 307         description => "The number of CPUs. Please use option -sockets instead.",
 308         minimum => 1,
 309         default => 1,
 310     },
 311     sockets => {
 312         optional => 1,
 313         type => 'integer',
 314         description => "The number of CPU sockets.",
 315         minimum => 1,
 316         default => 1,
 317     },
 318     cores => {
 319         optional => 1,
 320         type => 'integer',
 321         description => "The number of cores per socket.",
 322         minimum => 1,
 323         default => 1,
 324     },
 325     numa => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Enable/disable NUMA.",
 329         default => 0,
 330     },
 331     hugepages => {
 332         optional => 1,
 333         type => 'string',
 334         description => "Enable/disable hugepages memory.",
 335         enum => [qw(any 2 1024)],
 336     },
 337     vcpus => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Number of hotplugged vcpus.",
 341         minimum => 1,
 342         default => 0,
 343     },
 344     acpi => {
 345         optional => 1,
 346         type => 'boolean',
 347         description => "Enable/disable ACPI.",
 348         default => 1,
 349     },
 350     agent => {
 351         optional => 1,
 352         type => 'boolean',
 353         description => "Enable/disable Qemu GuestAgent.",
 354         default => 0,
 355     },
 356     kvm => {
 357         optional => 1,
 358         type => 'boolean',
 359         description => "Enable/disable KVM hardware virtualization.",
 360         default => 1,
 361     },
 362     tdf => {
 363         optional => 1,
 364         type => 'boolean',
 365         description => "Enable/disable time drift fix.",
 366         default => 0,
 367     },
 368     localtime => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 372     },
 373     freeze => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 377     },
 378     vga => {
 379         optional => 1,
 380         type => 'string',
 381         description => "Select the VGA type.",
 382         verbose_description => "Select the VGA type. If you want to use high resolution" .
 383             " modes (>= 1280x1024x16) then you should use the options " .
 384             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 385             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 386             "display sever. For win* OS you can select how many independent " .
 387             "displays you want, Linux guests can add displays them self. " .
 388             "You can also run without any graphic card, using a serial device" .
 389             " as terminal.",
 390         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 391     },
 392     watchdog => {
 393         optional => 1,
 394         type => 'string', format => 'pve-qm-watchdog',
 395         description => "Create a virtual hardware watchdog device.",
 396         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 397             " (by a guest action), the watchdog must be periodically polled " .
 398             "by an agent inside the guest or else the watchdog will reset " .
 399             "the guest (or execute the respective action specified)",
 400     },
 401     startdate => {
 402         optional => 1,
 403         type => 'string',
 404         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 405         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 406         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 407         default => 'now',
 408     },
 409     startup =>  get_standard_option('pve-startup-order'),
 410     template => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable Template.",
 414         default => 0,
 415     },
 416     args => {
 417         optional => 1,
 418         type => 'string',
 419         description => "Arbitrary arguments passed to kvm.",
 420         verbose_description => <<EODESCR,
 421 Arbitrary arguments passed to kvm, for example:
 422
 423 args: -no-reboot -no-hpet
 424
 425 NOTE: this option is for experts only.
 426 EODESCR
 427     },
 428     tablet => {
 429         optional => 1,
 430         type => 'boolean',
 431         default => 1,
 432         description => "Enable/disable the USB tablet device.",
 433         verbose_description => "Enable/disable the USB tablet device. This device is " .
 434             "usually needed to allow absolute mouse positioning with VNC. " .
 435             "Else the mouse runs out of sync with normal VNC clients. " .
 436             "If you're running lots of console-only guests on one host, " .
 437             "you may consider disabling this to save some context switches. " .
 438             "This is turned off by default if you use spice (-vga=qxl).",
 439     },
 440     migrate_speed => {
 441         optional => 1,
 442         type => 'integer',
 443         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 444         minimum => 0,
 445         default => 0,
 446     },
 447     migrate_downtime => {
 448         optional => 1,
 449         type => 'number',
 450         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 451         minimum => 0,
 452         default => 0.1,
 453     },
 454     cdrom => {
 455         optional => 1,
 456         type => 'string', format => 'pve-qm-ide',
 457         typetext => '<volume>',
 458         description => "This is an alias for option -ide2",
 459     },
 460     cpu => {
 461         optional => 1,
 462         description => "Emulated CPU type.",
 463         type => 'string',
 464         format => $cpu_fmt,
 465     },
 466     parent => get_standard_option('pve-snapshot-name', {
 467         optional => 1,
 468         description => "Parent snapshot name. This is used internally, and should not be modified.",
 469     }),
 470     snaptime => {
 471         optional => 1,
 472         description => "Timestamp for snapshots.",
 473         type => 'integer',
 474         minimum => 0,
 475     },
 476     vmstate => {
 477         optional => 1,
 478         type => 'string', format => 'pve-volume-id',
 479         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 480     },
 481     machine => {
 482         description => "Specific the Qemu machine type.",
 483         type => 'string',
 484         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 485         maxLength => 40,
 486         optional => 1,
 487     },
 488     smbios1 => {
 489         description => "Specify SMBIOS type 1 fields.",
 490         type => 'string', format => 'pve-qm-smbios1',
 491         maxLength => 256,
 492         optional => 1,
 493     },
 494     protection => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 498         default => 0,
 499     },
 500     bios => {
 501         optional => 1,
 502         type => 'string',
 503         enum => [ qw(seabios ovmf) ],
 504         description => "Select BIOS implementation.",
 505         default => 'seabios',
 506     },
 507 };
 508
 509 # what about other qemu settings ?
 510 #cpu => 'string',
 511 #machine => 'string',
 512 #fda => 'file',
 513 #fdb => 'file',
 514 #mtdblock => 'file',
 515 #sd => 'file',
 516 #pflash => 'file',
 517 #snapshot => 'bool',
 518 #bootp => 'file',
 519 ##tftp => 'dir',
 520 ##smb => 'dir',
 521 #kernel => 'file',
 522 #append => 'string',
 523 #initrd => 'file',
 524 ##soundhw => 'string',
 525
 526 while (my ($k, $v) = each %$confdesc) {
 527     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 528 }
 529
 530 my $MAX_IDE_DISKS = 4;
 531 my $MAX_SCSI_DISKS = 14;
 532 my $MAX_VIRTIO_DISKS = 16;
 533 my $MAX_SATA_DISKS = 6;
 534 my $MAX_USB_DEVICES = 5;
 535 my $MAX_NETS = 32;
 536 my $MAX_UNUSED_DISKS = 8;
 537 my $MAX_HOSTPCI_DEVICES = 4;
 538 my $MAX_SERIAL_PORTS = 4;
 539 my $MAX_PARALLEL_PORTS = 3;
 540 my $MAX_NUMA = 8;
 541
 542 my $numa_fmt = {
 543     cpus => {
 544         type => "string",
 545         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 546         description => "CPUs accessing this NUMA node.",
 547         format_description => "id[-id];...",
 548     },
 549     memory => {
 550         type => "number",
 551         description => "Amount of memory this NUMA node provides.",
 552         optional => 1,
 553     },
 554     hostnodes => {
 555         type => "string",
 556         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 557         description => "Host NUMA nodes to use.",
 558         format_description => "id[-id];...",
 559         optional => 1,
 560     },
 561     policy => {
 562         type => 'string',
 563         enum => [qw(preferred bind interleave)],
 564         description => "NUMA allocation policy.",
 565         optional => 1,
 566     },
 567 };
 568 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 569 my $numadesc = {
 570     optional => 1,
 571     type => 'string', format => $numa_fmt,
 572     description => "NUMA topology.",
 573 };
 574 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 575
 576 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 577     $confdesc->{"numa$i"} = $numadesc;
 578 }
 579
 580 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 581                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 582                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 583 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 584
 585 my $net_fmt_bridge_descr = <<__EOD__;
 586 Bridge to attach the network device to. The Proxmox VE standard bridge
 587 is called 'vmbr0'.
 588
 589 If you do not specify a bridge, we create a kvm user (NATed) network
 590 device, which provides DHCP and DNS services. The following addresses
 591 are used:
 592
 593  10.0.2.2   Gateway
 594  10.0.2.3   DNS Server
 595  10.0.2.4   SMB Server
 596
 597 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 598 __EOD__
 599
 600 my $net_fmt = {
 601     macaddr => {
 602         type => 'string',
 603         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 604         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 605         format_description => "XX:XX:XX:XX:XX:XX",
 606         optional => 1,
 607     },
 608     model => {
 609         type => 'string',
 610         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 611         enum => $nic_model_list,
 612         default_key => 1,
 613     },
 614     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 615     bridge => {
 616         type => 'string',
 617         description => $net_fmt_bridge_descr,
 618         format_description => 'bridge',
 619         optional => 1,
 620     },
 621     queues => {
 622         type => 'integer',
 623         minimum => 0, maximum => 16,
 624         description => 'Number of packet queues to be used on the device.',
 625         optional => 1,
 626     },
 627     rate => {
 628         type => 'number',
 629         minimum => 0,
 630         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 631         optional => 1,
 632     },
 633     tag => {
 634         type => 'integer',
 635         minimum => 1, maximum => 4094,
 636         description => 'VLAN tag to apply to packets on this interface.',
 637         optional => 1,
 638     },
 639     trunks => {
 640         type => 'string',
 641         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 642         description => 'VLAN trunks to pass through this interface.',
 643         format_description => 'vlanid[;vlanid...]',
 644         optional => 1,
 645     },
 646     firewall => {
 647         type => 'boolean',
 648         description => 'Whether this interface should be protected by the firewall.',
 649         optional => 1,
 650     },
 651     link_down => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be disconnected (like pulling the plug).',
 654         optional => 1,
 655     },
 656 };
 657
 658 my $netdesc = {
 659     optional => 1,
 660     type => 'string', format => $net_fmt,
 661     description => "Specify network devices.",
 662 };
 663
 664 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 665
 666 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 667     $confdesc->{"net$i"} = $netdesc;
 668 }
 669
 670 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 671 sub verify_volume_id_or_qm_path {
 672     my ($volid, $noerr) = @_;
 673
 674     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 675         return $volid;
 676     }
 677
 678     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 679     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 680     if ($@) {
 681         return undef if $noerr;
 682         die $@;
 683     }
 684     return $volid;
 685 }
 686
 687 my $drivename_hash;
 688
 689 my %drivedesc_base = (
 690     volume => { alias => 'file' },
 691     file => {
 692         type => 'string',
 693         format => 'pve-volume-id-or-qm-path',
 694         default_key => 1,
 695         format_description => 'volume',
 696         description => "The drive's backing volume.",
 697     },
 698     media => {
 699         type => 'string',
 700         enum => [qw(cdrom disk)],
 701         description => "The drive's media type.",
 702         default => 'disk',
 703         optional => 1
 704     },
 705     cyls => {
 706         type => 'integer',
 707         description => "Force the drive's physical geometry to have a specific cylinder count.",
 708         optional => 1
 709     },
 710     heads => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific head count.",
 713         optional => 1
 714     },
 715     secs => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific sector count.",
 718         optional => 1
 719     },
 720     trans => {
 721         type => 'string',
 722         enum => [qw(none lba auto)],
 723         description => "Force disk geometry bios translation mode.",
 724         optional => 1,
 725     },
 726     snapshot => {
 727         type => 'boolean',
 728         description => "Whether the drive should be included when making snapshots.",
 729         optional => 1,
 730     },
 731     cache => {
 732         type => 'string',
 733         enum => [qw(none writethrough writeback unsafe directsync)],
 734         description => "The drive's cache mode",
 735         optional => 1,
 736     },
 737     format => get_standard_option('pve-qm-image-format'),
 738     size => {
 739         type => 'string',
 740         format => 'disk-size',
 741         format_description => 'DiskSize',
 742         description => "Disk size. This is purely informational and has no effect.",
 743         optional => 1,
 744     },
 745     backup => {
 746         type => 'boolean',
 747         description => "Whether the drive should be included when making backups.",
 748         optional => 1,
 749     },
 750     replicate => {
 751         type => 'boolean',
 752         description => 'Whether the drive should considered for replication jobs.',
 753         optional => 1,
 754         default => 1,
 755     },
 756     rerror => {
 757         type => 'string',
 758         enum => [qw(ignore report stop)],
 759         description => 'Read error action.',
 760         optional => 1,
 761     },
 762     werror => {
 763         type => 'string',
 764         enum => [qw(enospc ignore report stop)],
 765         description => 'Write error action.',
 766         optional => 1,
 767     },
 768     aio => {
 769         type => 'string',
 770         enum => [qw(native threads)],
 771         description => 'AIO type to use.',
 772         optional => 1,
 773     },
 774     discard => {
 775         type => 'string',
 776         enum => [qw(ignore on)],
 777         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 778         optional => 1,
 779     },
 780     detect_zeroes => {
 781         type => 'boolean',
 782         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 783         optional => 1,
 784     },
 785     serial => {
 786         type => 'string',
 787         format => 'urlencoded',
 788         format_description => 'serial',
 789         maxLength => 20*3, # *3 since it's %xx url enoded
 790         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 791         optional => 1,
 792     }
 793 );
 794
 795 my %iothread_fmt = ( iothread => {
 796         type => 'boolean',
 797         description => "Whether to use iothreads for this drive",
 798         optional => 1,
 799 });
 800
 801 my %model_fmt = (
 802     model => {
 803         type => 'string',
 804         format => 'urlencoded',
 805         format_description => 'model',
 806         maxLength => 40*3, # *3 since it's %xx url enoded
 807         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 808         optional => 1,
 809     },
 810 );
 811
 812 my %queues_fmt = (
 813     queues => {
 814         type => 'integer',
 815         description => "Number of queues.",
 816         minimum => 2,
 817         optional => 1
 818     }
 819 );
 820
 821 my %scsiblock_fmt = (
 822     scsiblock => {
 823         type => 'boolean',
 824         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 825         optional => 1,
 826         default => 0,
 827     },
 828 );
 829
 830 my $add_throttle_desc = sub {
 831     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 832     my $d = {
 833         type => $type,
 834         format_description => $unit,
 835         description => "Maximum $what in $longunit.",
 836         optional => 1,
 837     };
 838     $d->{minimum} = $minimum if defined($minimum);
 839     $drivedesc_base{$key} = $d;
 840 };
 841 # throughput: (leaky bucket)
 842 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 843 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 844 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 845 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 846 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 847 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 848 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 849 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 850 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 851
 852 # pools: (pool of IO before throttling starts taking effect)
 853 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 854 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 855 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 856 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 857 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 858 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 859
 860 # burst lengths
 861 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 862 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 863 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 864 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 865 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 866 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 867
 868 # legacy support
 869 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 870 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 871 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 872 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 873
 874 my $ide_fmt = {
 875     %drivedesc_base,
 876     %model_fmt,
 877 };
 878 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 879
 880 my $idedesc = {
 881     optional => 1,
 882     type => 'string', format => $ide_fmt,
 883     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 884 };
 885 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 886
 887 my $scsi_fmt = {
 888     %drivedesc_base,
 889     %iothread_fmt,
 890     %queues_fmt,
 891     %scsiblock_fmt,
 892 };
 893 my $scsidesc = {
 894     optional => 1,
 895     type => 'string', format => $scsi_fmt,
 896     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 897 };
 898 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 899
 900 my $sata_fmt = {
 901     %drivedesc_base,
 902 };
 903 my $satadesc = {
 904     optional => 1,
 905     type => 'string', format => $sata_fmt,
 906     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 907 };
 908 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 909
 910 my $virtio_fmt = {
 911     %drivedesc_base,
 912     %iothread_fmt,
 913 };
 914 my $virtiodesc = {
 915     optional => 1,
 916     type => 'string', format => $virtio_fmt,
 917     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 918 };
 919 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 920
 921 my $alldrive_fmt = {
 922     %drivedesc_base,
 923     %iothread_fmt,
 924     %model_fmt,
 925     %queues_fmt,
 926     %scsiblock_fmt,
 927 };
 928
 929 my $efidisk_fmt = {
 930     volume => { alias => 'file' },
 931     file => {
 932         type => 'string',
 933         format => 'pve-volume-id-or-qm-path',
 934         default_key => 1,
 935         format_description => 'volume',
 936         description => "The drive's backing volume.",
 937     },
 938     format => get_standard_option('pve-qm-image-format'),
 939     size => {
 940         type => 'string',
 941         format => 'disk-size',
 942         format_description => 'DiskSize',
 943         description => "Disk size. This is purely informational and has no effect.",
 944         optional => 1,
 945     },
 946 };
 947
 948 my $efidisk_desc = {
 949     optional => 1,
 950     type => 'string', format => $efidisk_fmt,
 951     description => "Configure a Disk for storing EFI vars",
 952 };
 953
 954 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 955
 956 my $usb_fmt = {
 957     host => {
 958         default_key => 1,
 959         type => 'string', format => 'pve-qm-usb-device',
 960         format_description => 'HOSTUSBDEVICE|spice',
 961         description => <<EODESCR,
 962 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 963
 964  'bus-port(.port)*' (decimal numbers) or
 965  'vendor_id:product_id' (hexadeciaml numbers) or
 966  'spice'
 967
 968 You can use the 'lsusb -t' command to list existing usb devices.
 969
 970 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 971
 972 The value 'spice' can be used to add a usb redirection devices for spice.
 973 EODESCR
 974     },
 975     usb3 => {
 976         optional => 1,
 977         type => 'boolean',
 978         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 979         default => 0,
 980     },
 981 };
 982
 983 my $usbdesc = {
 984     optional => 1,
 985     type => 'string', format => $usb_fmt,
 986     description => "Configure an USB device (n is 0 to 4).",
 987 };
 988 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 989
 990 # NOTE: the match-groups of this regex are used in parse_hostpci
 991 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 992 my $hostpci_fmt = {
 993     host => {
 994         default_key => 1,
 995         type => 'string',
 996         pattern => qr/$PCIRE(;$PCIRE)*/,
 997         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 998         description => <<EODESCR,
 999 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1000 of PCI virtual functions of the host. HOSTPCIID syntax is:
1001
1002 'bus:dev.func' (hexadecimal numbers)
1003
1004 You can us the 'lspci' command to list existing PCI devices.
1005 EODESCR
1006     },
1007     rombar => {
1008         type => 'boolean',
1009         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1010         optional => 1,
1011         default => 1,
1012     },
1013     romfile => {
1014         type => 'string',
1015         pattern => '[^,;]+',
1016         format_description => 'string',
1017         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1018         optional => 1,
1019     },
1020     pcie => {
1021         type => 'boolean',
1022         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1023         optional => 1,
1024         default => 0,
1025     },
1026     'x-vga' => {
1027         type => 'boolean',
1028         description =>  "Enable vfio-vga device support.",
1029         optional => 1,
1030         default => 0,
1031     },
1032 };
1033 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1034
1035 my $hostpcidesc = {
1036         optional => 1,
1037         type => 'string', format => 'pve-qm-hostpci',
1038         description => "Map host PCI devices into guest.",
1039         verbose_description =>  <<EODESCR,
1040 Map host PCI devices into guest.
1041
1042 NOTE: This option allows direct access to host hardware. So it is no longer
1043 possible to migrate such machines - use with special care.
1044
1045 CAUTION: Experimental! User reported problems with this option.
1046 EODESCR
1047 };
1048 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1049
1050 my $serialdesc = {
1051         optional => 1,
1052         type => 'string',
1053         pattern => '(/dev/.+|socket)',
1054         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1055         verbose_description =>  <<EODESCR,
1056 Create a serial device inside the VM (n is 0 to 3), and pass through a
1057 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1058 host side (use 'qm terminal' to open a terminal connection).
1059
1060 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1061
1062 CAUTION: Experimental! User reported problems with this option.
1063 EODESCR
1064 };
1065
1066 my $paralleldesc= {
1067         optional => 1,
1068         type => 'string',
1069         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1070         description =>  "Map host parallel devices (n is 0 to 2).",
1071         verbose_description =>  <<EODESCR,
1072 Map host parallel devices (n is 0 to 2).
1073
1074 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1075
1076 CAUTION: Experimental! User reported problems with this option.
1077 EODESCR
1078 };
1079
1080 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1081     $confdesc->{"parallel$i"} = $paralleldesc;
1082 }
1083
1084 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1085     $confdesc->{"serial$i"} = $serialdesc;
1086 }
1087
1088 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1089     $confdesc->{"hostpci$i"} = $hostpcidesc;
1090 }
1091
1092 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1093     $drivename_hash->{"ide$i"} = 1;
1094     $confdesc->{"ide$i"} = $idedesc;
1095 }
1096
1097 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1098     $drivename_hash->{"sata$i"} = 1;
1099     $confdesc->{"sata$i"} = $satadesc;
1100 }
1101
1102 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1103     $drivename_hash->{"scsi$i"} = 1;
1104     $confdesc->{"scsi$i"} = $scsidesc ;
1105 }
1106
1107 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1108     $drivename_hash->{"virtio$i"} = 1;
1109     $confdesc->{"virtio$i"} = $virtiodesc;
1110 }
1111
1112 $drivename_hash->{efidisk0} = 1;
1113 $confdesc->{efidisk0} = $efidisk_desc;
1114
1115 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1116     $confdesc->{"usb$i"} = $usbdesc;
1117 }
1118
1119 my $unuseddesc = {
1120     optional => 1,
1121     type => 'string', format => 'pve-volume-id',
1122     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1123 };
1124
1125 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1126     $confdesc->{"unused$i"} = $unuseddesc;
1127 }
1128
1129 my $kvm_api_version = 0;
1130
1131 sub kvm_version {
1132
1133     return $kvm_api_version if $kvm_api_version;
1134
1135     my $fh = IO::File->new("</dev/kvm") ||
1136         return 0;
1137
1138     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1139         $kvm_api_version = $v;
1140     }
1141
1142     $fh->close();
1143
1144     return  $kvm_api_version;
1145 }
1146
1147 my $kvm_user_version;
1148
1149 sub kvm_user_version {
1150
1151     return $kvm_user_version if $kvm_user_version;
1152
1153     $kvm_user_version = 'unknown';
1154
1155     my $code = sub {
1156         my $line = shift;
1157         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1158             $kvm_user_version = $2;
1159         }
1160     };
1161
1162     eval { run_command("kvm -version", outfunc => $code); };
1163     warn $@ if $@;
1164
1165     return $kvm_user_version;
1166
1167 }
1168
1169 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1170
1171 sub valid_drive_names {
1172     # order is important - used to autoselect boot disk
1173     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1174             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1175             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1176             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1177             'efidisk0');
1178 }
1179
1180 sub is_valid_drivename {
1181     my $dev = shift;
1182
1183     return defined($drivename_hash->{$dev});
1184 }
1185
1186 sub option_exists {
1187     my $key = shift;
1188     return defined($confdesc->{$key});
1189 }
1190
1191 sub nic_models {
1192     return $nic_model_list;
1193 }
1194
1195 sub os_list_description {
1196
1197     return {
1198         other => 'Other',
1199         wxp => 'Windows XP',
1200         w2k => 'Windows 2000',
1201         w2k3 =>, 'Windows 2003',
1202         w2k8 => 'Windows 2008',
1203         wvista => 'Windows Vista',
1204         win7 => 'Windows 7',
1205         win8 => 'Windows 8/2012',
1206         win10 => 'Windows 10/2016',
1207         l24 => 'Linux 2.4',
1208         l26 => 'Linux 2.6',
1209     };
1210 }
1211
1212 my $cdrom_path;
1213
1214 sub get_cdrom_path {
1215
1216     return  $cdrom_path if $cdrom_path;
1217
1218     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1219     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1220     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1221 }
1222
1223 sub get_iso_path {
1224     my ($storecfg, $vmid, $cdrom) = @_;
1225
1226     if ($cdrom eq 'cdrom') {
1227         return get_cdrom_path();
1228     } elsif ($cdrom eq 'none') {
1229         return '';
1230     } elsif ($cdrom =~ m|^/|) {
1231         return $cdrom;
1232     } else {
1233         return PVE::Storage::path($storecfg, $cdrom);
1234     }
1235 }
1236
1237 # try to convert old style file names to volume IDs
1238 sub filename_to_volume_id {
1239     my ($vmid, $file, $media) = @_;
1240
1241     if (!($file eq 'none' || $file eq 'cdrom' ||
1242           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1243
1244         return undef if $file =~ m|/|;
1245
1246         if ($media && $media eq 'cdrom') {
1247             $file = "local:iso/$file";
1248         } else {
1249             $file = "local:$vmid/$file";
1250         }
1251     }
1252
1253     return $file;
1254 }
1255
1256 sub verify_media_type {
1257     my ($opt, $vtype, $media) = @_;
1258
1259     return if !$media;
1260
1261     my $etype;
1262     if ($media eq 'disk') {
1263         $etype = 'images';
1264     } elsif ($media eq 'cdrom') {
1265         $etype = 'iso';
1266     } else {
1267         die "internal error";
1268     }
1269
1270     return if ($vtype eq $etype);
1271
1272     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1273 }
1274
1275 sub cleanup_drive_path {
1276     my ($opt, $storecfg, $drive) = @_;
1277
1278     # try to convert filesystem paths to volume IDs
1279
1280     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1281         ($drive->{file} !~ m|^/dev/.+|) &&
1282         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1283         ($drive->{file} !~ m/^\d+$/)) {
1284         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1285         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1286         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1287         verify_media_type($opt, $vtype, $drive->{media});
1288         $drive->{file} = $volid;
1289     }
1290
1291     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1292 }
1293
1294 sub parse_hotplug_features {
1295     my ($data) = @_;
1296
1297     my $res = {};
1298
1299     return $res if $data eq '0';
1300
1301     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1302
1303     foreach my $feature (PVE::Tools::split_list($data)) {
1304         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1305             $res->{$1} = 1;
1306         } else {
1307             die "invalid hotplug feature '$feature'\n";
1308         }
1309     }
1310     return $res;
1311 }
1312
1313 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1314 sub pve_verify_hotplug_features {
1315     my ($value, $noerr) = @_;
1316
1317     return $value if parse_hotplug_features($value);
1318
1319     return undef if $noerr;
1320
1321     die "unable to parse hotplug option\n";
1322 }
1323
1324 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1325 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1326 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1327 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1328 #        [,iothread=on][,serial=serial][,model=model]
1329
1330 sub parse_drive {
1331     my ($key, $data) = @_;
1332
1333     my ($interface, $index);
1334
1335     if ($key =~ m/^([^\d]+)(\d+)$/) {
1336         $interface = $1;
1337         $index = $2;
1338     } else {
1339         return undef;
1340     }
1341
1342     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1343                                      : $confdesc->{$key}->{format};
1344     if (!$desc) {
1345         warn "invalid drive key: $key\n";
1346         return undef;
1347     }
1348     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1349     return undef if !$res;
1350     $res->{interface} = $interface;
1351     $res->{index} = $index;
1352
1353     my $error = 0;
1354     foreach my $opt (qw(bps bps_rd bps_wr)) {
1355         if (my $bps = defined(delete $res->{$opt})) {
1356             if (defined($res->{"m$opt"})) {
1357                 warn "both $opt and m$opt specified\n";
1358                 ++$error;
1359                 next;
1360             }
1361             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1362         }
1363     }
1364
1365     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1366     for my $requirement (
1367         [mbps_max => 'mbps'],
1368         [mbps_rd_max => 'mbps_rd'],
1369         [mbps_wr_max => 'mbps_wr'],
1370         [miops_max => 'miops'],
1371         [miops_rd_max => 'miops_rd'],
1372         [miops_wr_max => 'miops_wr'],
1373         [bps_max_length => 'mbps_max'],
1374         [bps_rd_max_length => 'mbps_rd_max'],
1375         [bps_wr_max_length => 'mbps_wr_max'],
1376         [iops_max_length => 'iops_max'],
1377         [iops_rd_max_length => 'iops_rd_max'],
1378         [iops_wr_max_length => 'iops_wr_max']) {
1379         my ($option, $requires) = @$requirement;
1380         if ($res->{$option} && !$res->{$requires}) {
1381             warn "$option requires $requires\n";
1382             ++$error;
1383         }
1384     }
1385
1386     return undef if $error;
1387
1388     return undef if $res->{mbps_rd} && $res->{mbps};
1389     return undef if $res->{mbps_wr} && $res->{mbps};
1390     return undef if $res->{iops_rd} && $res->{iops};
1391     return undef if $res->{iops_wr} && $res->{iops};
1392
1393     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1394         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1395         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1396         return undef if $res->{interface} eq 'virtio';
1397     }
1398
1399     if (my $size = $res->{size}) {
1400         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1401     }
1402
1403     return $res;
1404 }
1405
1406 sub print_drive {
1407     my ($vmid, $drive) = @_;
1408     my $data = { %$drive };
1409     delete $data->{$_} for qw(index interface);
1410     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1411 }
1412
1413 sub scsi_inquiry {
1414     my($fh, $noerr) = @_;
1415
1416     my $SG_IO = 0x2285;
1417     my $SG_GET_VERSION_NUM = 0x2282;
1418
1419     my $versionbuf = "\x00" x 8;
1420     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1421     if (!$ret) {
1422         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1423         return undef;
1424     }
1425     my $version = unpack("I", $versionbuf);
1426     if ($version < 30000) {
1427         die "scsi generic interface too old\n"  if !$noerr;
1428         return undef;
1429     }
1430
1431     my $buf = "\x00" x 36;
1432     my $sensebuf = "\x00" x 8;
1433     my $cmd = pack("C x3 C x1", 0x12, 36);
1434
1435     # see /usr/include/scsi/sg.h
1436     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1437
1438     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1439                       length($sensebuf), 0, length($buf), $buf,
1440                       $cmd, $sensebuf, 6000);
1441
1442     $ret = ioctl($fh, $SG_IO, $packet);
1443     if (!$ret) {
1444         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1445         return undef;
1446     }
1447
1448     my @res = unpack($sg_io_hdr_t, $packet);
1449     if ($res[17] || $res[18]) {
1450         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1451         return undef;
1452     }
1453
1454     my $res = {};
1455     (my $byte0, my $byte1, $res->{vendor},
1456      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1457
1458     $res->{removable} = $byte1 & 128 ? 1 : 0;
1459     $res->{type} = $byte0 & 31;
1460
1461     return $res;
1462 }
1463
1464 sub path_is_scsi {
1465     my ($path) = @_;
1466
1467     my $fh = IO::File->new("+<$path") || return undef;
1468     my $res = scsi_inquiry($fh, 1);
1469     close($fh);
1470
1471     return $res;
1472 }
1473
1474 sub machine_type_is_q35 {
1475     my ($conf) = @_;
1476
1477     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1478 }
1479
1480 sub print_tabletdevice_full {
1481     my ($conf) = @_;
1482
1483     my $q35 = machine_type_is_q35($conf);
1484
1485     # we use uhci for old VMs because tablet driver was buggy in older qemu
1486     my $usbbus = $q35 ? "ehci" : "uhci";
1487
1488     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1489 }
1490
1491 sub print_drivedevice_full {
1492     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1493
1494     my $device = '';
1495     my $maxdev = 0;
1496
1497     if ($drive->{interface} eq 'virtio') {
1498         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1499         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1500         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1501     } elsif ($drive->{interface} eq 'scsi') {
1502
1503         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1504         my $unit = $drive->{index} % $maxdev;
1505         my $devicetype = 'hd';
1506         my $path = '';
1507         if (drive_is_cdrom($drive)) {
1508             $devicetype = 'cd';
1509         } else {
1510             if ($drive->{file} =~ m|^/|) {
1511                 $path = $drive->{file};
1512                 if (my $info = path_is_scsi($path)) {
1513                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1514                         $devicetype = 'block';
1515                     } elsif ($info->{type} == 1) { # tape
1516                         $devicetype = 'generic';
1517                     }
1518                 }
1519             } else {
1520                  $path = PVE::Storage::path($storecfg, $drive->{file});
1521             }
1522
1523             if($path =~ m/^iscsi\:\/\//){
1524                 $devicetype = 'generic';
1525             }
1526         }
1527
1528         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1529             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1530         } else {
1531             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1532         }
1533
1534     } elsif ($drive->{interface} eq 'ide'){
1535         $maxdev = 2;
1536         my $controller = int($drive->{index} / $maxdev);
1537         my $unit = $drive->{index} % $maxdev;
1538         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1539
1540         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1541         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1542             $model = URI::Escape::uri_unescape($model);
1543             $device .= ",model=$model";
1544         }
1545     } elsif ($drive->{interface} eq 'sata'){
1546         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1547         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1548         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1549     } elsif ($drive->{interface} eq 'usb') {
1550         die "implement me";
1551         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1552     } else {
1553         die "unsupported interface type";
1554     }
1555
1556     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1557
1558     return $device;
1559 }
1560
1561 sub get_initiator_name {
1562     my $initiator;
1563
1564     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1565     while (defined(my $line = <$fh>)) {
1566         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1567         $initiator = $1;
1568         last;
1569     }
1570     $fh->close();
1571
1572     return $initiator;
1573 }
1574
1575 sub print_drive_full {
1576     my ($storecfg, $vmid, $drive) = @_;
1577
1578     my $path;
1579     my $volid = $drive->{file};
1580     my $format;
1581
1582     if (drive_is_cdrom($drive)) {
1583         $path = get_iso_path($storecfg, $vmid, $volid);
1584     } else {
1585         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1586         if ($storeid) {
1587             $path = PVE::Storage::path($storecfg, $volid);
1588             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1589             $format = qemu_img_format($scfg, $volname);
1590         } else {
1591             $path = $volid;
1592             $format = "raw";
1593         }
1594    }
1595
1596     my $opts = '';
1597     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1598     foreach my $o (@qemu_drive_options) {
1599         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1600     }
1601     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1602         my ($dir, $qmpname) = @$type;
1603         if (my $v = $drive->{"mbps$dir"}) {
1604             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1605         }
1606         if (my $v = $drive->{"mbps${dir}_max"}) {
1607             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1608         }
1609         if (my $v = $drive->{"bps${dir}_max_length"}) {
1610             $opts .= ",throttling.bps$qmpname-max-length=$v";
1611         }
1612         if (my $v = $drive->{"iops${dir}"}) {
1613             $opts .= ",throttling.iops$qmpname=$v";
1614         }
1615         if (my $v = $drive->{"iops${dir}_max"}) {
1616             $opts .= ",throttling.iops$qmpname=-max$v";
1617         }
1618         if (my $v = $drive->{"iops${dir}_max_length"}) {
1619             $opts .= ",throttling.iops$qmpname=-max-length$v";
1620         }
1621     }
1622
1623     if (my $serial = $drive->{serial}) {
1624         $serial = URI::Escape::uri_unescape($serial);
1625         $opts .= ",serial=$serial";
1626     }
1627
1628     $opts .= ",format=$format" if $format && !$drive->{format};
1629
1630     my $cache_direct = 0;
1631
1632     if (my $cache = $drive->{cache}) {
1633         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1634     } elsif (!drive_is_cdrom($drive)) {
1635         $opts .= ",cache=none";
1636         $cache_direct = 1;
1637     }
1638
1639     # aio native works only with O_DIRECT
1640     if (!$drive->{aio}) {
1641         if($cache_direct) {
1642             $opts .= ",aio=native";
1643         } else {
1644             $opts .= ",aio=threads";
1645         }
1646     }
1647
1648     if (!drive_is_cdrom($drive)) {
1649         my $detectzeroes;
1650         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1651             $detectzeroes = 'off';
1652         } elsif ($drive->{discard}) {
1653             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1654         } else {
1655             # This used to be our default with discard not being specified:
1656             $detectzeroes = 'on';
1657         }
1658         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1659     }
1660
1661     my $pathinfo = $path ? "file=$path," : '';
1662
1663     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1664 }
1665
1666 sub print_netdevice_full {
1667     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1668
1669     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1670
1671     my $device = $net->{model};
1672     if ($net->{model} eq 'virtio') {
1673          $device = 'virtio-net-pci';
1674      };
1675
1676     my $pciaddr = print_pci_addr("$netid", $bridges);
1677     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1678     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1679         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1680         my $vectors = $net->{queues} * 2 + 2;
1681         $tmpstr .= ",vectors=$vectors,mq=on";
1682     }
1683     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1684
1685     if ($use_old_bios_files) {
1686         my $romfile;
1687         if ($device eq 'virtio-net-pci') {
1688             $romfile = 'pxe-virtio.rom';
1689         } elsif ($device eq 'e1000') {
1690             $romfile = 'pxe-e1000.rom';
1691         } elsif ($device eq 'ne2k') {
1692             $romfile = 'pxe-ne2k_pci.rom';
1693         } elsif ($device eq 'pcnet') {
1694             $romfile = 'pxe-pcnet.rom';
1695         } elsif ($device eq 'rtl8139') {
1696             $romfile = 'pxe-rtl8139.rom';
1697         }
1698         $tmpstr .= ",romfile=$romfile" if $romfile;
1699     }
1700
1701     return $tmpstr;
1702 }
1703
1704 sub print_netdev_full {
1705     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1706
1707     my $i = '';
1708     if ($netid =~ m/^net(\d+)$/) {
1709         $i = int($1);
1710     }
1711
1712     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1713
1714     my $ifname = "tap${vmid}i$i";
1715
1716     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1717     die "interface name '$ifname' is too long (max 15 character)\n"
1718         if length($ifname) >= 16;
1719
1720     my $vhostparam = '';
1721     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1722
1723     my $vmname = $conf->{name} || "vm$vmid";
1724
1725     my $netdev = "";
1726     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1727
1728     if ($net->{bridge}) {
1729         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1730     } else {
1731         $netdev = "type=user,id=$netid,hostname=$vmname";
1732     }
1733
1734     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1735
1736     return $netdev;
1737 }
1738
1739
1740 sub print_cpu_device {
1741     my ($conf, $id) = @_;
1742
1743     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1744     my $cpu = $nokvm ? "qemu64" : "kvm64";
1745     if (my $cputype = $conf->{cpu}) {
1746         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1747             or die "Cannot parse cpu description: $cputype\n";
1748         $cpu = $cpuconf->{cputype};
1749     }
1750
1751     my $cores = $conf->{cores} || 1;
1752
1753     my $current_core = ($id - 1) % $cores;
1754     my $current_socket = int(($id - 1 - $current_core)/$cores);
1755
1756     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1757 }
1758
1759 sub drive_is_cdrom {
1760     my ($drive) = @_;
1761
1762     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1763
1764 }
1765
1766 sub parse_number_sets {
1767     my ($set) = @_;
1768     my $res = [];
1769     foreach my $part (split(/;/, $set)) {
1770         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1771             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1772             push @$res, [ $1, $2 ];
1773         } else {
1774             die "invalid range: $part\n";
1775         }
1776     }
1777     return $res;
1778 }
1779
1780 sub parse_numa {
1781     my ($data) = @_;
1782
1783     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1784     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1785     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1786     return $res;
1787 }
1788
1789 sub parse_hostpci {
1790     my ($value) = @_;
1791
1792     return undef if !$value;
1793
1794     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1795
1796     my @idlist = split(/;/, $res->{host});
1797     delete $res->{host};
1798     foreach my $id (@idlist) {
1799         if ($id =~ /^$PCIRE$/) {
1800             if (defined($2)) {
1801                 push @{$res->{pciid}}, { id => $1, function => $2 };
1802             } else {
1803                 my $pcidevices = lspci($1);
1804                 $res->{pciid} = $pcidevices->{$1};
1805             }
1806         } else {
1807             # should have been caught by parse_property_string already
1808             die "failed to parse PCI id: $id\n";
1809         }
1810     }
1811     return $res;
1812 }
1813
1814 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1815 sub parse_net {
1816     my ($data) = @_;
1817
1818     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1819     if ($@) {
1820         warn $@;
1821         return undef;
1822     }
1823     if (!defined($res->{macaddr})) {
1824         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1825         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1826     }
1827     return $res;
1828 }
1829
1830 sub print_net {
1831     my $net = shift;
1832
1833     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1834 }
1835
1836 sub add_random_macs {
1837     my ($settings) = @_;
1838
1839     foreach my $opt (keys %$settings) {
1840         next if $opt !~ m/^net(\d+)$/;
1841         my $net = parse_net($settings->{$opt});
1842         next if !$net;
1843         $settings->{$opt} = print_net($net);
1844     }
1845 }
1846
1847 sub vm_is_volid_owner {
1848     my ($storecfg, $vmid, $volid) = @_;
1849
1850     if ($volid !~  m|^/|) {
1851         my ($path, $owner);
1852         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1853         if ($owner && ($owner == $vmid)) {
1854             return 1;
1855         }
1856     }
1857
1858     return undef;
1859 }
1860
1861 sub split_flagged_list {
1862     my $text = shift || '';
1863     $text =~ s/[,;]/ /g;
1864     $text =~ s/^\s+//;
1865     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1866 }
1867
1868 sub join_flagged_list {
1869     my ($how, $lst) = @_;
1870     join $how, map { $lst->{$_} . $_ } keys %$lst;
1871 }
1872
1873 sub vmconfig_delete_pending_option {
1874     my ($conf, $key, $force) = @_;
1875
1876     delete $conf->{pending}->{$key};
1877     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1878     $pending_delete_hash->{$key} = $force ? '!' : '';
1879     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1880 }
1881
1882 sub vmconfig_undelete_pending_option {
1883     my ($conf, $key) = @_;
1884
1885     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1886     delete $pending_delete_hash->{$key};
1887
1888     if (%$pending_delete_hash) {
1889         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1890     } else {
1891         delete $conf->{pending}->{delete};
1892     }
1893 }
1894
1895 sub vmconfig_register_unused_drive {
1896     my ($storecfg, $vmid, $conf, $drive) = @_;
1897
1898     if (!drive_is_cdrom($drive)) {
1899         my $volid = $drive->{file};
1900         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1901             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1902         }
1903     }
1904 }
1905
1906 sub vmconfig_cleanup_pending {
1907     my ($conf) = @_;
1908
1909     # remove pending changes when nothing changed
1910     my $changes;
1911     foreach my $opt (keys %{$conf->{pending}}) {
1912         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1913             $changes = 1;
1914             delete $conf->{pending}->{$opt};
1915         }
1916     }
1917
1918     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1919     my $pending_delete_hash = {};
1920     while (my ($opt, $force) = each %$current_delete_hash) {
1921         if (defined($conf->{$opt})) {
1922             $pending_delete_hash->{$opt} = $force;
1923         } else {
1924             $changes = 1;
1925         }
1926     }
1927
1928     if (%$pending_delete_hash) {
1929         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1930     } else {
1931         delete $conf->{pending}->{delete};
1932     }
1933
1934     return $changes;
1935 }
1936
1937 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1938 my $smbios1_fmt = {
1939     uuid => {
1940         type => 'string',
1941         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1942         format_description => 'UUID',
1943         description => "Set SMBIOS1 UUID.",
1944         optional => 1,
1945     },
1946     version => {
1947         type => 'string',
1948         pattern => '\S+',
1949         format_description => 'string',
1950         description => "Set SMBIOS1 version.",
1951         optional => 1,
1952     },
1953     serial => {
1954         type => 'string',
1955         pattern => '\S+',
1956         format_description => 'string',
1957         description => "Set SMBIOS1 serial number.",
1958         optional => 1,
1959     },
1960     manufacturer => {
1961         type => 'string',
1962         pattern => '\S+',
1963         format_description => 'string',
1964         description => "Set SMBIOS1 manufacturer.",
1965         optional => 1,
1966     },
1967     product => {
1968         type => 'string',
1969         pattern => '\S+',
1970         format_description => 'string',
1971         description => "Set SMBIOS1 product ID.",
1972         optional => 1,
1973     },
1974     sku => {
1975         type => 'string',
1976         pattern => '\S+',
1977         format_description => 'string',
1978         description => "Set SMBIOS1 SKU string.",
1979         optional => 1,
1980     },
1981     family => {
1982         type => 'string',
1983         pattern => '\S+',
1984         format_description => 'string',
1985         description => "Set SMBIOS1 family string.",
1986         optional => 1,
1987     },
1988 };
1989
1990 sub parse_smbios1 {
1991     my ($data) = @_;
1992
1993     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1994     warn $@ if $@;
1995     return $res;
1996 }
1997
1998 sub print_smbios1 {
1999     my ($smbios1) = @_;
2000     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2001 }
2002
2003 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2004
2005 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2006 sub verify_bootdisk {
2007     my ($value, $noerr) = @_;
2008
2009     return $value if is_valid_drivename($value);
2010
2011     return undef if $noerr;
2012
2013     die "invalid boot disk '$value'\n";
2014 }
2015
2016 sub parse_watchdog {
2017     my ($value) = @_;
2018
2019     return undef if !$value;
2020
2021     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2022     warn $@ if $@;
2023     return $res;
2024 }
2025
2026 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2027 sub verify_usb_device {
2028     my ($value, $noerr) = @_;
2029
2030     return $value if parse_usb_device($value);
2031
2032     return undef if $noerr;
2033
2034     die "unable to parse usb device\n";
2035 }
2036
2037 # add JSON properties for create and set function
2038 sub json_config_properties {
2039     my $prop = shift;
2040
2041     foreach my $opt (keys %$confdesc) {
2042         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2043         $prop->{$opt} = $confdesc->{$opt};
2044     }
2045
2046     return $prop;
2047 }
2048
2049 sub check_type {
2050     my ($key, $value) = @_;
2051
2052     die "unknown setting '$key'\n" if !$confdesc->{$key};
2053
2054     my $type = $confdesc->{$key}->{type};
2055
2056     if (!defined($value)) {
2057         die "got undefined value\n";
2058     }
2059
2060     if ($value =~ m/[\n\r]/) {
2061         die "property contains a line feed\n";
2062     }
2063
2064     if ($type eq 'boolean') {
2065         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2066         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2067         die "type check ('boolean') failed - got '$value'\n";
2068     } elsif ($type eq 'integer') {
2069         return int($1) if $value =~ m/^(\d+)$/;
2070         die "type check ('integer') failed - got '$value'\n";
2071     } elsif ($type eq 'number') {
2072         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2073         die "type check ('number') failed - got '$value'\n";
2074     } elsif ($type eq 'string') {
2075         if (my $fmt = $confdesc->{$key}->{format}) {
2076             PVE::JSONSchema::check_format($fmt, $value);
2077             return $value;
2078         }
2079         $value =~ s/^\"(.*)\"$/$1/;
2080         return $value;
2081     } else {
2082         die "internal error"
2083     }
2084 }
2085
2086 sub check_iommu_support{
2087     #fixme : need to check IOMMU support
2088     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2089
2090     my $iommu=1;
2091     return $iommu;
2092
2093 }
2094
2095 sub touch_config {
2096     my ($vmid) = @_;
2097
2098     my $conf = PVE::QemuConfig->config_file($vmid);
2099     utime undef, undef, $conf;
2100 }
2101
2102 sub destroy_vm {
2103     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2104
2105     my $conffile = PVE::QemuConfig->config_file($vmid);
2106
2107     my $conf = PVE::QemuConfig->load_config($vmid);
2108
2109     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2110
2111     # only remove disks owned by this VM
2112     foreach_drive($conf, sub {
2113         my ($ds, $drive) = @_;
2114
2115         return if drive_is_cdrom($drive);
2116
2117         my $volid = $drive->{file};
2118
2119         return if !$volid || $volid =~ m|^/|;
2120
2121         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2122         return if !$path || !$owner || ($owner != $vmid);
2123
2124         eval {
2125             PVE::Storage::vdisk_free($storecfg, $volid);
2126         };
2127         warn "Could not remove disk '$volid', check manually: $@" if $@;
2128
2129     });
2130
2131     if ($keep_empty_config) {
2132         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2133     } else {
2134         unlink $conffile;
2135     }
2136
2137     # also remove unused disk
2138     eval {
2139         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2140
2141         eval {
2142             PVE::Storage::foreach_volid($dl, sub {
2143                 my ($volid, $sid, $volname, $d) = @_;
2144                 PVE::Storage::vdisk_free($storecfg, $volid);
2145             });
2146         };
2147         warn $@ if $@;
2148
2149     };
2150     warn $@ if $@;
2151 }
2152
2153 sub parse_vm_config {
2154     my ($filename, $raw) = @_;
2155
2156     return undef if !defined($raw);
2157
2158     my $res = {
2159         digest => Digest::SHA::sha1_hex($raw),
2160         snapshots => {},
2161         pending => {},
2162     };
2163
2164     $filename =~ m|/qemu-server/(\d+)\.conf$|
2165         || die "got strange filename '$filename'";
2166
2167     my $vmid = $1;
2168
2169     my $conf = $res;
2170     my $descr;
2171     my $section = '';
2172
2173     my @lines = split(/\n/, $raw);
2174     foreach my $line (@lines) {
2175         next if $line =~ m/^\s*$/;
2176
2177         if ($line =~ m/^\[PENDING\]\s*$/i) {
2178             $section = 'pending';
2179             if (defined($descr)) {
2180                 $descr =~ s/\s+$//;
2181                 $conf->{description} = $descr;
2182             }
2183             $descr = undef;
2184             $conf = $res->{$section} = {};
2185             next;
2186
2187         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2188             $section = $1;
2189             if (defined($descr)) {
2190                 $descr =~ s/\s+$//;
2191                 $conf->{description} = $descr;
2192             }
2193             $descr = undef;
2194             $conf = $res->{snapshots}->{$section} = {};
2195             next;
2196         }
2197
2198         if ($line =~ m/^\#(.*)\s*$/) {
2199             $descr = '' if !defined($descr);
2200             $descr .= PVE::Tools::decode_text($1) . "\n";
2201             next;
2202         }
2203
2204         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2205             $descr = '' if !defined($descr);
2206             $descr .= PVE::Tools::decode_text($2);
2207         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2208             $conf->{snapstate} = $1;
2209         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2210             my $key = $1;
2211             my $value = $2;
2212             $conf->{$key} = $value;
2213         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2214             my $value = $1;
2215             if ($section eq 'pending') {
2216                 $conf->{delete} = $value; # we parse this later
2217             } else {
2218                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2219             }
2220         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2221             my $key = $1;
2222             my $value = $2;
2223             eval { $value = check_type($key, $value); };
2224             if ($@) {
2225                 warn "vm $vmid - unable to parse value of '$key' - $@";
2226             } else {
2227                 $key = 'ide2' if $key eq 'cdrom';
2228                 my $fmt = $confdesc->{$key}->{format};
2229                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2230                     my $v = parse_drive($key, $value);
2231                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2232                         $v->{file} = $volid;
2233                         $value = print_drive($vmid, $v);
2234                     } else {
2235                         warn "vm $vmid - unable to parse value of '$key'\n";
2236                         next;
2237                     }
2238                 }
2239
2240                 $conf->{$key} = $value;
2241             }
2242         }
2243     }
2244
2245     if (defined($descr)) {
2246         $descr =~ s/\s+$//;
2247         $conf->{description} = $descr;
2248     }
2249     delete $res->{snapstate}; # just to be sure
2250
2251     return $res;
2252 }
2253
2254 sub write_vm_config {
2255     my ($filename, $conf) = @_;
2256
2257     delete $conf->{snapstate}; # just to be sure
2258
2259     if ($conf->{cdrom}) {
2260         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2261         $conf->{ide2} = $conf->{cdrom};
2262         delete $conf->{cdrom};
2263     }
2264
2265     # we do not use 'smp' any longer
2266     if ($conf->{sockets}) {
2267         delete $conf->{smp};
2268     } elsif ($conf->{smp}) {
2269         $conf->{sockets} = $conf->{smp};
2270         delete $conf->{cores};
2271         delete $conf->{smp};
2272     }
2273
2274     my $used_volids = {};
2275
2276     my $cleanup_config = sub {
2277         my ($cref, $pending, $snapname) = @_;
2278
2279         foreach my $key (keys %$cref) {
2280             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2281                 $key eq 'snapstate' || $key eq 'pending';
2282             my $value = $cref->{$key};
2283             if ($key eq 'delete') {
2284                 die "propertry 'delete' is only allowed in [PENDING]\n"
2285                     if !$pending;
2286                 # fixme: check syntax?
2287                 next;
2288             }
2289             eval { $value = check_type($key, $value); };
2290             die "unable to parse value of '$key' - $@" if $@;
2291
2292             $cref->{$key} = $value;
2293
2294             if (!$snapname && is_valid_drivename($key)) {
2295                 my $drive = parse_drive($key, $value);
2296                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2297             }
2298         }
2299     };
2300
2301     &$cleanup_config($conf);
2302
2303     &$cleanup_config($conf->{pending}, 1);
2304
2305     foreach my $snapname (keys %{$conf->{snapshots}}) {
2306         die "internal error" if $snapname eq 'pending';
2307         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2308     }
2309
2310     # remove 'unusedX' settings if we re-add a volume
2311     foreach my $key (keys %$conf) {
2312         my $value = $conf->{$key};
2313         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2314             delete $conf->{$key};
2315         }
2316     }
2317
2318     my $generate_raw_config = sub {
2319         my ($conf, $pending) = @_;
2320
2321         my $raw = '';
2322
2323         # add description as comment to top of file
2324         if (defined(my $descr = $conf->{description})) {
2325             if ($descr) {
2326                 foreach my $cl (split(/\n/, $descr)) {
2327                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2328                 }
2329             } else {
2330                 $raw .= "#\n" if $pending;
2331             }
2332         }
2333
2334         foreach my $key (sort keys %$conf) {
2335             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2336             $raw .= "$key: $conf->{$key}\n";
2337         }
2338         return $raw;
2339     };
2340
2341     my $raw = &$generate_raw_config($conf);
2342
2343     if (scalar(keys %{$conf->{pending}})){
2344         $raw .= "\n[PENDING]\n";
2345         $raw .= &$generate_raw_config($conf->{pending}, 1);
2346     }
2347
2348     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2349         $raw .= "\n[$snapname]\n";
2350         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2351     }
2352
2353     return $raw;
2354 }
2355
2356 sub load_defaults {
2357
2358     my $res = {};
2359
2360     # we use static defaults from our JSON schema configuration
2361     foreach my $key (keys %$confdesc) {
2362         if (defined(my $default = $confdesc->{$key}->{default})) {
2363             $res->{$key} = $default;
2364         }
2365     }
2366
2367     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2368     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2369
2370     return $res;
2371 }
2372
2373 sub config_list {
2374     my $vmlist = PVE::Cluster::get_vmlist();
2375     my $res = {};
2376     return $res if !$vmlist || !$vmlist->{ids};
2377     my $ids = $vmlist->{ids};
2378
2379     foreach my $vmid (keys %$ids) {
2380         my $d = $ids->{$vmid};
2381         next if !$d->{node} || $d->{node} ne $nodename;
2382         next if !$d->{type} || $d->{type} ne 'qemu';
2383         $res->{$vmid}->{exists} = 1;
2384     }
2385     return $res;
2386 }
2387
2388 # test if VM uses local resources (to prevent migration)
2389 sub check_local_resources {
2390     my ($conf, $noerr) = @_;
2391
2392     my $loc_res = 0;
2393
2394     $loc_res = 1 if $conf->{hostusb}; # old syntax
2395     $loc_res = 1 if $conf->{hostpci}; # old syntax
2396
2397     foreach my $k (keys %$conf) {
2398         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2399         # sockets are safe: they will recreated be on the target side post-migrate
2400         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2401         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2402     }
2403
2404     die "VM uses local resources\n" if $loc_res && !$noerr;
2405
2406     return $loc_res;
2407 }
2408
2409 # check if used storages are available on all nodes (use by migrate)
2410 sub check_storage_availability {
2411     my ($storecfg, $conf, $node) = @_;
2412
2413     foreach_drive($conf, sub {
2414         my ($ds, $drive) = @_;
2415
2416         my $volid = $drive->{file};
2417         return if !$volid;
2418
2419         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2420         return if !$sid;
2421
2422         # check if storage is available on both nodes
2423         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2424         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2425    });
2426 }
2427
2428 # list nodes where all VM images are available (used by has_feature API)
2429 sub shared_nodes {
2430     my ($conf, $storecfg) = @_;
2431
2432     my $nodelist = PVE::Cluster::get_nodelist();
2433     my $nodehash = { map { $_ => 1 } @$nodelist };
2434     my $nodename = PVE::INotify::nodename();
2435
2436     foreach_drive($conf, sub {
2437         my ($ds, $drive) = @_;
2438
2439         my $volid = $drive->{file};
2440         return if !$volid;
2441
2442         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2443         if ($storeid) {
2444             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2445             if ($scfg->{disable}) {
2446                 $nodehash = {};
2447             } elsif (my $avail = $scfg->{nodes}) {
2448                 foreach my $node (keys %$nodehash) {
2449                     delete $nodehash->{$node} if !$avail->{$node};
2450                 }
2451             } elsif (!$scfg->{shared}) {
2452                 foreach my $node (keys %$nodehash) {
2453                     delete $nodehash->{$node} if $node ne $nodename
2454                 }
2455             }
2456         }
2457     });
2458
2459     return $nodehash
2460 }
2461
2462 sub check_cmdline {
2463     my ($pidfile, $pid) = @_;
2464
2465     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2466     if (defined($fh)) {
2467         my $line = <$fh>;
2468         $fh->close;
2469         return undef if !$line;
2470         my @param = split(/\0/, $line);
2471
2472         my $cmd = $param[0];
2473         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2474
2475         for (my $i = 0; $i < scalar (@param); $i++) {
2476             my $p = $param[$i];
2477             next if !$p;
2478             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2479                 my $p = $param[$i+1];
2480                 return 1 if $p && ($p eq $pidfile);
2481                 return undef;
2482             }
2483         }
2484     }
2485     return undef;
2486 }
2487
2488 sub check_running {
2489     my ($vmid, $nocheck, $node) = @_;
2490
2491     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2492
2493     die "unable to find configuration file for VM $vmid - no such machine\n"
2494         if !$nocheck && ! -f $filename;
2495
2496     my $pidfile = pidfile_name($vmid);
2497
2498     if (my $fd = IO::File->new("<$pidfile")) {
2499         my $st = stat($fd);
2500         my $line = <$fd>;
2501         close($fd);
2502
2503         my $mtime = $st->mtime;
2504         if ($mtime > time()) {
2505             warn "file '$filename' modified in future\n";
2506         }
2507
2508         if ($line =~ m/^(\d+)$/) {
2509             my $pid = $1;
2510             if (check_cmdline($pidfile, $pid)) {
2511                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2512                     return $pid;
2513                 }
2514             }
2515         }
2516     }
2517
2518     return undef;
2519 }
2520
2521 sub vzlist {
2522
2523     my $vzlist = config_list();
2524
2525     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2526
2527     while (defined(my $de = $fd->read)) {
2528         next if $de !~ m/^(\d+)\.pid$/;
2529         my $vmid = $1;
2530         next if !defined($vzlist->{$vmid});
2531         if (my $pid = check_running($vmid)) {
2532             $vzlist->{$vmid}->{pid} = $pid;
2533         }
2534     }
2535
2536     return $vzlist;
2537 }
2538
2539 sub disksize {
2540     my ($storecfg, $conf) = @_;
2541
2542     my $bootdisk = $conf->{bootdisk};
2543     return undef if !$bootdisk;
2544     return undef if !is_valid_drivename($bootdisk);
2545
2546     return undef if !$conf->{$bootdisk};
2547
2548     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2549     return undef if !defined($drive);
2550
2551     return undef if drive_is_cdrom($drive);
2552
2553     my $volid = $drive->{file};
2554     return undef if !$volid;
2555
2556     return $drive->{size};
2557 }
2558
2559 my $last_proc_pid_stat;
2560
2561 # get VM status information
2562 # This must be fast and should not block ($full == false)
2563 # We only query KVM using QMP if $full == true (this can be slow)
2564 sub vmstatus {
2565     my ($opt_vmid, $full) = @_;
2566
2567     my $res = {};
2568
2569     my $storecfg = PVE::Storage::config();
2570
2571     my $list = vzlist();
2572     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2573
2574     my $cpucount = $cpuinfo->{cpus} || 1;
2575
2576     foreach my $vmid (keys %$list) {
2577         next if $opt_vmid && ($vmid ne $opt_vmid);
2578
2579         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2580         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2581
2582         my $d = {};
2583         $d->{pid} = $list->{$vmid}->{pid};
2584
2585         # fixme: better status?
2586         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2587
2588         my $size = disksize($storecfg, $conf);
2589         if (defined($size)) {
2590             $d->{disk} = 0; # no info available
2591             $d->{maxdisk} = $size;
2592         } else {
2593             $d->{disk} = 0;
2594             $d->{maxdisk} = 0;
2595         }
2596
2597         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2598         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2599         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2600
2601         $d->{name} = $conf->{name} || "VM $vmid";
2602         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2603
2604         if ($conf->{balloon}) {
2605             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2606             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2607         }
2608
2609         $d->{uptime} = 0;
2610         $d->{cpu} = 0;
2611         $d->{mem} = 0;
2612
2613         $d->{netout} = 0;
2614         $d->{netin} = 0;
2615
2616         $d->{diskread} = 0;
2617         $d->{diskwrite} = 0;
2618
2619         $d->{template} = PVE::QemuConfig->is_template($conf);
2620
2621         $res->{$vmid} = $d;
2622     }
2623
2624     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2625     foreach my $dev (keys %$netdev) {
2626         next if $dev !~ m/^tap([1-9]\d*)i/;
2627         my $vmid = $1;
2628         my $d = $res->{$vmid};
2629         next if !$d;
2630
2631         $d->{netout} += $netdev->{$dev}->{receive};
2632         $d->{netin} += $netdev->{$dev}->{transmit};
2633
2634         if ($full) {
2635             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2636             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2637         }
2638
2639     }
2640
2641     my $ctime = gettimeofday;
2642
2643     foreach my $vmid (keys %$list) {
2644
2645         my $d = $res->{$vmid};
2646         my $pid = $d->{pid};
2647         next if !$pid;
2648
2649         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2650         next if !$pstat; # not running
2651
2652         my $used = $pstat->{utime} + $pstat->{stime};
2653
2654         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2655
2656         if ($pstat->{vsize}) {
2657             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2658         }
2659
2660         my $old = $last_proc_pid_stat->{$pid};
2661         if (!$old) {
2662             $last_proc_pid_stat->{$pid} = {
2663                 time => $ctime,
2664                 used => $used,
2665                 cpu => 0,
2666             };
2667             next;
2668         }
2669
2670         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2671
2672         if ($dtime > 1000) {
2673             my $dutime = $used -  $old->{used};
2674
2675             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2676             $last_proc_pid_stat->{$pid} = {
2677                 time => $ctime,
2678                 used => $used,
2679                 cpu => $d->{cpu},
2680             };
2681         } else {
2682             $d->{cpu} = $old->{cpu};
2683         }
2684     }
2685
2686     return $res if !$full;
2687
2688     my $qmpclient = PVE::QMPClient->new();
2689
2690     my $ballooncb = sub {
2691         my ($vmid, $resp) = @_;
2692
2693         my $info = $resp->{'return'};
2694         return if !$info->{max_mem};
2695
2696         my $d = $res->{$vmid};
2697
2698         # use memory assigned to VM
2699         $d->{maxmem} = $info->{max_mem};
2700         $d->{balloon} = $info->{actual};
2701
2702         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2703             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2704             $d->{freemem} = $info->{free_mem};
2705         }
2706
2707         $d->{ballooninfo} = $info;
2708     };
2709
2710     my $blockstatscb = sub {
2711         my ($vmid, $resp) = @_;
2712         my $data = $resp->{'return'} || [];
2713         my $totalrdbytes = 0;
2714         my $totalwrbytes = 0;
2715
2716         for my $blockstat (@$data) {
2717             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2718             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2719
2720             $blockstat->{device} =~ s/drive-//;
2721             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2722         }
2723         $res->{$vmid}->{diskread} = $totalrdbytes;
2724         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2725     };
2726
2727     my $statuscb = sub {
2728         my ($vmid, $resp) = @_;
2729
2730         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2731         # this fails if ballon driver is not loaded, so this must be
2732         # the last commnand (following command are aborted if this fails).
2733         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2734
2735         my $status = 'unknown';
2736         if (!defined($status = $resp->{'return'}->{status})) {
2737             warn "unable to get VM status\n";
2738             return;
2739         }
2740
2741         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2742     };
2743
2744     foreach my $vmid (keys %$list) {
2745         next if $opt_vmid && ($vmid ne $opt_vmid);
2746         next if !$res->{$vmid}->{pid}; # not running
2747         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2748     }
2749
2750     $qmpclient->queue_execute(undef, 2);
2751
2752     foreach my $vmid (keys %$list) {
2753         next if $opt_vmid && ($vmid ne $opt_vmid);
2754         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2755     }
2756
2757     return $res;
2758 }
2759
2760 sub foreach_drive {
2761     my ($conf, $func, @param) = @_;
2762
2763     foreach my $ds (valid_drive_names()) {
2764         next if !defined($conf->{$ds});
2765
2766         my $drive = parse_drive($ds, $conf->{$ds});
2767         next if !$drive;
2768
2769         &$func($ds, $drive, @param);
2770     }
2771 }
2772
2773 sub foreach_volid {
2774     my ($conf, $func, @param) = @_;
2775
2776     my $volhash = {};
2777
2778     my $test_volid = sub {
2779         my ($volid, $is_cdrom, $replicate) = @_;
2780
2781         return if !$volid;
2782
2783         $volhash->{$volid}->{cdrom} //= 1;
2784         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2785
2786         $volhash->{$volid}->{replicate} //= 0;
2787         $volhash->{$volid}->{replicate} = 1 if $replicate;
2788     };
2789
2790     foreach_drive($conf, sub {
2791         my ($ds, $drive) = @_;
2792         &$test_volid($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1);
2793     });
2794
2795     foreach my $snapname (keys %{$conf->{snapshots}}) {
2796         my $snap = $conf->{snapshots}->{$snapname};
2797         &$test_volid($snap->{vmstate}, 0, 1);
2798         foreach_drive($snap, sub {
2799             my ($ds, $drive) = @_;
2800             &$test_volid($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1);
2801         });
2802     }
2803
2804     foreach my $volid (keys %$volhash) {
2805         &$func($volid, $volhash->{$volid}, @param);
2806     }
2807 }
2808
2809 sub vga_conf_has_spice {
2810     my ($vga) = @_;
2811
2812     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2813
2814     return $1 || 1;
2815 }
2816
2817 sub config_to_command {
2818     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2819
2820     my $cmd = [];
2821     my $globalFlags = [];
2822     my $machineFlags = [];
2823     my $rtcFlags = [];
2824     my $cpuFlags = [];
2825     my $devices = [];
2826     my $pciaddr = '';
2827     my $bridges = {};
2828     my $kvmver = kvm_user_version();
2829     my $vernum = 0; # unknown
2830     my $ostype = $conf->{ostype};
2831     my $winversion = windows_version($ostype);
2832
2833     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2834         $vernum = $1*1000000+$2*1000;
2835     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2836         $vernum = $1*1000000+$2*1000+$3;
2837     }
2838
2839     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2840
2841     my $have_ovz = -f '/proc/vz/vestat';
2842
2843     my $q35 = machine_type_is_q35($conf);
2844     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2845     my $machine_type = $forcemachine || $conf->{machine};
2846     my $use_old_bios_files = undef;
2847     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2848
2849     my $cpuunits = defined($conf->{cpuunits}) ?
2850             $conf->{cpuunits} : $defaults->{cpuunits};
2851
2852     push @$cmd, '/usr/bin/kvm';
2853
2854     push @$cmd, '-id', $vmid;
2855
2856     my $use_virtio = 0;
2857
2858     my $qmpsocket = qmp_socket($vmid);
2859     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2860     push @$cmd, '-mon', "chardev=qmp,mode=control";
2861
2862
2863     push @$cmd, '-pidfile' , pidfile_name($vmid);
2864
2865     push @$cmd, '-daemonize';
2866
2867     if ($conf->{smbios1}) {
2868         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2869     }
2870
2871     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2872         my $ovmfbase;
2873
2874         # prefer the OVMF_CODE variant
2875         if (-f $OVMF_CODE) {
2876             $ovmfbase = $OVMF_CODE;
2877         } elsif (-f $OVMF_IMG) {
2878             $ovmfbase = $OVMF_IMG;
2879         }
2880
2881         die "no uefi base img found\n" if !$ovmfbase;
2882         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2883
2884         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2885             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2886             my $format = $d->{format} // 'raw';
2887             my $path;
2888             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2889             if ($storeid) {
2890                 $path = PVE::Storage::path($storecfg, $d->{file});
2891                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2892                 $format = qemu_img_format($scfg, $volname);
2893             } else {
2894                 $path = $d->{file};
2895                 $format = "raw";
2896             }
2897             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2898         } elsif ($ovmfbase eq $OVMF_CODE) {
2899             warn "using uefi without permanent efivars disk\n";
2900             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2901             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2902             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2903         } else {
2904             # if the base img is not OVMF_CODE, we do not have to bother
2905             # to create/use a vars image, since it will not be used anyway
2906             # this can only happen if someone manually deletes the OVMF_CODE image
2907             # or has an old pve-qemu-kvm version installed.
2908             # both should not happen, but we ignore it here
2909         }
2910     }
2911
2912
2913     # add usb controllers
2914     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2915     push @$devices, @usbcontrollers if @usbcontrollers;
2916     my $vga = $conf->{vga};
2917
2918     my $qxlnum = vga_conf_has_spice($vga);
2919     $vga = 'qxl' if $qxlnum;
2920
2921     if (!$vga) {
2922         $vga = $winversion >= 6 ? 'std' : 'cirrus';
2923     }
2924
2925     # enable absolute mouse coordinates (needed by vnc)
2926     my $tablet;
2927     if (defined($conf->{tablet})) {
2928         $tablet = $conf->{tablet};
2929     } else {
2930         $tablet = $defaults->{tablet};
2931         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2932         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2933     }
2934
2935     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2936
2937     my $kvm_off = 0;
2938     my $gpu_passthrough;
2939
2940     # host pci devices
2941     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2942         my $d = parse_hostpci($conf->{"hostpci$i"});
2943         next if !$d;
2944
2945         my $pcie = $d->{pcie};
2946         if($pcie){
2947             die "q35 machine model is not enabled" if !$q35;
2948             $pciaddr = print_pcie_addr("hostpci$i");
2949         }else{
2950             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2951         }
2952
2953         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2954         my $romfile = $d->{romfile};
2955
2956         my $xvga = '';
2957         if ($d->{'x-vga'}) {
2958             $xvga = ',x-vga=on';
2959             $kvm_off = 1;
2960             $vga = 'none';
2961             $gpu_passthrough = 1;
2962
2963             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2964                 $xvga = "";
2965             }
2966         }
2967         my $pcidevices = $d->{pciid};
2968         my $multifunction = 1 if @$pcidevices > 1;
2969
2970         my $j=0;
2971         foreach my $pcidevice (@$pcidevices) {
2972
2973             my $id = "hostpci$i";
2974             $id .= ".$j" if $multifunction;
2975             my $addr = $pciaddr;
2976             $addr .= ".$j" if $multifunction;
2977             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2978
2979             if($j == 0){
2980                 $devicestr .= "$rombar$xvga";
2981                 $devicestr .= ",multifunction=on" if $multifunction;
2982                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
2983             }
2984
2985             push @$devices, '-device', $devicestr;
2986             $j++;
2987         }
2988     }
2989
2990     # usb devices
2991     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2992     push @$devices, @usbdevices if @usbdevices;
2993     # serial devices
2994     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2995         if (my $path = $conf->{"serial$i"}) {
2996             if ($path eq 'socket') {
2997                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2998                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2999                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3000             } else {
3001                 die "no such serial device\n" if ! -c $path;
3002                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3003                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3004             }
3005         }
3006     }
3007
3008     # parallel devices
3009     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3010         if (my $path = $conf->{"parallel$i"}) {
3011             die "no such parallel device\n" if ! -c $path;
3012             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3013             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3014             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3015         }
3016     }
3017
3018     my $vmname = $conf->{name} || "vm$vmid";
3019
3020     push @$cmd, '-name', $vmname;
3021
3022     my $sockets = 1;
3023     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3024     $sockets = $conf->{sockets} if  $conf->{sockets};
3025
3026     my $cores = $conf->{cores} || 1;
3027
3028     my $maxcpus = $sockets * $cores;
3029
3030     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3031
3032     my $allowed_vcpus = $cpuinfo->{cpus};
3033
3034     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3035         if ($allowed_vcpus < $maxcpus);
3036
3037     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3038
3039         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3040         for (my $i = 2; $i <= $vcpus; $i++)  {
3041             my $cpustr = print_cpu_device($conf,$i);
3042             push @$cmd, '-device', $cpustr;
3043         }
3044
3045     } else {
3046
3047         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3048     }
3049     push @$cmd, '-nodefaults';
3050
3051     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3052
3053     my $bootindex_hash = {};
3054     my $i = 1;
3055     foreach my $o (split(//, $bootorder)) {
3056         $bootindex_hash->{$o} = $i*100;
3057         $i++;
3058     }
3059
3060     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3061
3062     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3063
3064     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3065
3066     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3067
3068     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3069         my $socket = vnc_socket($vmid);
3070         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3071     } else {
3072         push @$cmd, '-nographic';
3073     }
3074
3075     # time drift fix
3076     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3077
3078     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3079     my $useLocaltime = $conf->{localtime};
3080
3081     if ($winversion >= 5) { # windows
3082         $useLocaltime = 1 if !defined($conf->{localtime});
3083
3084         # use time drift fix when acpi is enabled
3085         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3086             $tdf = 1 if !defined($conf->{tdf});
3087         }
3088     }
3089
3090     if ($winversion >= 6) {
3091         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3092         push @$cmd, '-no-hpet';
3093     }
3094
3095     push @$rtcFlags, 'driftfix=slew' if $tdf;
3096
3097     if ($nokvm) {
3098         push @$machineFlags, 'accel=tcg';
3099     } else {
3100         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3101     }
3102
3103     if ($machine_type) {
3104         push @$machineFlags, "type=${machine_type}";
3105     }
3106
3107     if ($conf->{startdate}) {
3108         push @$rtcFlags, "base=$conf->{startdate}";
3109     } elsif ($useLocaltime) {
3110         push @$rtcFlags, 'base=localtime';
3111     }
3112
3113     my $cpu = $nokvm ? "qemu64" : "kvm64";
3114     if (my $cputype = $conf->{cpu}) {
3115         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3116             or die "Cannot parse cpu description: $cputype\n";
3117         $cpu = $cpuconf->{cputype};
3118         $kvm_off = 1 if $cpuconf->{hidden};
3119     }
3120
3121     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3122
3123     push @$cpuFlags , '-x2apic'
3124         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3125
3126     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3127
3128     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3129
3130     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3131
3132         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3133         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3134     }
3135
3136     add_hyperv_enlighments($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $conf->{bios}, $gpu_passthrough);
3137
3138     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3139
3140     push @$cpuFlags, 'kvm=off' if $kvm_off;
3141
3142     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3143         die "internal error"; # should not happen
3144
3145     push @$cpuFlags, "vendor=${cpu_vendor}"
3146         if $cpu_vendor ne 'default';
3147
3148     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3149
3150     push @$cmd, '-cpu', $cpu;
3151
3152     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3153
3154     push @$cmd, '-S' if $conf->{freeze};
3155
3156     # set keyboard layout
3157     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3158     push @$cmd, '-k', $kb if $kb;
3159
3160     # enable sound
3161     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3162     #push @$cmd, '-soundhw', 'es1370';
3163     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3164
3165     if($conf->{agent}) {
3166         my $qgasocket = qmp_socket($vmid, 1);
3167         my $pciaddr = print_pci_addr("qga0", $bridges);
3168         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3169         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3170         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3171     }
3172
3173     my $spice_port;
3174
3175     if ($qxlnum) {
3176         if ($qxlnum > 1) {
3177             if ($winversion){
3178                 for(my $i = 1; $i < $qxlnum; $i++){
3179                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3180                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3181                 }
3182             } else {
3183                 # assume other OS works like Linux
3184                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3185                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3186             }
3187         }
3188
3189         my $pciaddr = print_pci_addr("spice", $bridges);
3190
3191         my $nodename = PVE::INotify::nodename();
3192         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3193         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3194         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3195         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3196         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3197
3198         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3199
3200         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3201         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3202         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3203     }
3204
3205     # enable balloon by default, unless explicitly disabled
3206     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3207         $pciaddr = print_pci_addr("balloon0", $bridges);
3208         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3209     }
3210
3211     if ($conf->{watchdog}) {
3212         my $wdopts = parse_watchdog($conf->{watchdog});
3213         $pciaddr = print_pci_addr("watchdog", $bridges);
3214         my $watchdog = $wdopts->{model} || 'i6300esb';
3215         push @$devices, '-device', "$watchdog$pciaddr";
3216         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3217     }
3218
3219     my $vollist = [];
3220     my $scsicontroller = {};
3221     my $ahcicontroller = {};
3222     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3223
3224     # Add iscsi initiator name if available
3225     if (my $initiator = get_initiator_name()) {
3226         push @$devices, '-iscsi', "initiator-name=$initiator";
3227     }
3228
3229     foreach_drive($conf, sub {
3230         my ($ds, $drive) = @_;
3231
3232         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3233             push @$vollist, $drive->{file};
3234         }
3235
3236         $use_virtio = 1 if $ds =~ m/^virtio/;
3237
3238         if (drive_is_cdrom ($drive)) {
3239             if ($bootindex_hash->{d}) {
3240                 $drive->{bootindex} = $bootindex_hash->{d};
3241                 $bootindex_hash->{d} += 1;
3242             }
3243         } else {
3244             if ($bootindex_hash->{c}) {
3245                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3246                 $bootindex_hash->{c} += 1;
3247             }
3248         }
3249
3250         if($drive->{interface} eq 'virtio'){
3251            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3252         }
3253
3254         if ($drive->{interface} eq 'scsi') {
3255
3256             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3257
3258             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3259             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3260
3261             my $iothread = '';
3262             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3263                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3264                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3265             } elsif ($drive->{iothread}) {
3266                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3267             }
3268
3269             my $queues = '';
3270             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3271                 $queues = ",num_queues=$drive->{queues}";
3272             }
3273
3274             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3275             $scsicontroller->{$controller}=1;
3276         }
3277
3278         if ($drive->{interface} eq 'sata') {
3279            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3280            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3281            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3282            $ahcicontroller->{$controller}=1;
3283         }
3284
3285         if ($drive->{interface} eq 'efidisk') {
3286             # this will be added somewhere else
3287             return;
3288         }
3289
3290         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3291         push @$devices, '-drive',$drive_cmd;
3292         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3293     });
3294
3295     for (my $i = 0; $i < $MAX_NETS; $i++) {
3296          next if !$conf->{"net$i"};
3297          my $d = parse_net($conf->{"net$i"});
3298          next if !$d;
3299
3300          $use_virtio = 1 if $d->{model} eq 'virtio';
3301
3302          if ($bootindex_hash->{n}) {
3303             $d->{bootindex} = $bootindex_hash->{n};
3304             $bootindex_hash->{n} += 1;
3305          }
3306
3307          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3308          push @$devices, '-netdev', $netdevfull;
3309
3310          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3311          push @$devices, '-device', $netdevicefull;
3312     }
3313
3314     if (!$q35) {
3315         # add pci bridges
3316         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3317            $bridges->{1} = 1;
3318            $bridges->{2} = 1;
3319         }
3320
3321         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3322
3323         while (my ($k, $v) = each %$bridges) {
3324             $pciaddr = print_pci_addr("pci.$k");
3325             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3326         }
3327     }
3328
3329     # add custom args
3330     if ($conf->{args}) {
3331         my $aa = PVE::Tools::split_args($conf->{args});
3332         push @$cmd, @$aa;
3333     }
3334
3335     push @$cmd, @$devices;
3336     push @$cmd, '-rtc', join(',', @$rtcFlags)
3337         if scalar(@$rtcFlags);
3338     push @$cmd, '-machine', join(',', @$machineFlags)
3339         if scalar(@$machineFlags);
3340     push @$cmd, '-global', join(',', @$globalFlags)
3341         if scalar(@$globalFlags);
3342
3343     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3344 }
3345
3346 sub vnc_socket {
3347     my ($vmid) = @_;
3348     return "${var_run_tmpdir}/$vmid.vnc";
3349 }
3350
3351 sub spice_port {
3352     my ($vmid) = @_;
3353
3354     my $res = vm_mon_cmd($vmid, 'query-spice');
3355
3356     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3357 }
3358
3359 sub qmp_socket {
3360     my ($vmid, $qga) = @_;
3361     my $sockettype = $qga ? 'qga' : 'qmp';
3362     return "${var_run_tmpdir}/$vmid.$sockettype";
3363 }
3364
3365 sub pidfile_name {
3366     my ($vmid) = @_;
3367     return "${var_run_tmpdir}/$vmid.pid";
3368 }
3369
3370 sub vm_devices_list {
3371     my ($vmid) = @_;
3372
3373     my $res = vm_mon_cmd($vmid, 'query-pci');
3374     my $devices = {};
3375     foreach my $pcibus (@$res) {
3376         foreach my $device (@{$pcibus->{devices}}) {
3377             next if !$device->{'qdev_id'};
3378             if ($device->{'pci_bridge'}) {
3379                 $devices->{$device->{'qdev_id'}} = 1;
3380                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3381                     next if !$bridge_device->{'qdev_id'};
3382                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3383                     $devices->{$device->{'qdev_id'}}++;
3384                 }
3385             } else {
3386                 $devices->{$device->{'qdev_id'}} = 1;
3387             }
3388         }
3389     }
3390
3391     my $resblock = vm_mon_cmd($vmid, 'query-block');
3392     foreach my $block (@$resblock) {
3393         if($block->{device} =~ m/^drive-(\S+)/){
3394                 $devices->{$1} = 1;
3395         }
3396     }
3397
3398     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3399     foreach my $mice (@$resmice) {
3400         if ($mice->{name} eq 'QEMU HID Tablet') {
3401             $devices->{tablet} = 1;
3402             last;
3403         }
3404     }
3405
3406     # for usb devices there is no query-usb
3407     # but we can iterate over the entries in
3408     # qom-list path=/machine/peripheral
3409     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3410     foreach my $per (@$resperipheral) {
3411         if ($per->{name} =~ m/^usb\d+$/) {
3412             $devices->{$per->{name}} = 1;
3413         }
3414     }
3415
3416     return $devices;
3417 }
3418
3419 sub vm_deviceplug {
3420     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3421
3422     my $q35 = machine_type_is_q35($conf);
3423
3424     my $devices_list = vm_devices_list($vmid);
3425     return 1 if defined($devices_list->{$deviceid});
3426
3427     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3428
3429     if ($deviceid eq 'tablet') {
3430
3431         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3432
3433     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3434
3435         die "usb hotplug currently not reliable\n";
3436         # since we can't reliably hot unplug all added usb devices
3437         # and usb passthrough disables live migration
3438         # we disable usb hotplugging for now
3439         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3440
3441     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3442
3443         qemu_iothread_add($vmid, $deviceid, $device);
3444
3445         qemu_driveadd($storecfg, $vmid, $device);
3446         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3447
3448         qemu_deviceadd($vmid, $devicefull);
3449         eval { qemu_deviceaddverify($vmid, $deviceid); };
3450         if (my $err = $@) {
3451             eval { qemu_drivedel($vmid, $deviceid); };
3452             warn $@ if $@;
3453             die $err;
3454         }
3455
3456     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3457
3458
3459         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3460         my $pciaddr = print_pci_addr($deviceid);
3461         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3462
3463         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3464
3465         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3466             qemu_iothread_add($vmid, $deviceid, $device);
3467             $devicefull .= ",iothread=iothread-$deviceid";
3468         }
3469
3470         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3471             $devicefull .= ",num_queues=$device->{queues}";
3472         }
3473
3474         qemu_deviceadd($vmid, $devicefull);
3475         qemu_deviceaddverify($vmid, $deviceid);
3476
3477     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3478
3479         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3480         qemu_driveadd($storecfg, $vmid, $device);
3481
3482         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3483         eval { qemu_deviceadd($vmid, $devicefull); };
3484         if (my $err = $@) {
3485             eval { qemu_drivedel($vmid, $deviceid); };
3486             warn $@ if $@;
3487             die $err;
3488         }
3489
3490     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3491
3492         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3493
3494         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3495         my $use_old_bios_files = undef;
3496         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3497
3498         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3499         qemu_deviceadd($vmid, $netdevicefull);
3500         eval { qemu_deviceaddverify($vmid, $deviceid); };
3501         if (my $err = $@) {
3502             eval { qemu_netdevdel($vmid, $deviceid); };
3503             warn $@ if $@;
3504             die $err;
3505         }
3506
3507     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3508
3509         my $bridgeid = $2;
3510         my $pciaddr = print_pci_addr($deviceid);
3511         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3512
3513         qemu_deviceadd($vmid, $devicefull);
3514         qemu_deviceaddverify($vmid, $deviceid);
3515
3516     } else {
3517         die "can't hotplug device '$deviceid'\n";
3518     }
3519
3520     return 1;
3521 }
3522
3523 # fixme: this should raise exceptions on error!
3524 sub vm_deviceunplug {
3525     my ($vmid, $conf, $deviceid) = @_;
3526
3527     my $devices_list = vm_devices_list($vmid);
3528     return 1 if !defined($devices_list->{$deviceid});
3529
3530     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3531
3532     if ($deviceid eq 'tablet') {
3533
3534         qemu_devicedel($vmid, $deviceid);
3535
3536     } elsif ($deviceid =~ m/^usb\d+$/) {
3537
3538         die "usb hotplug currently not reliable\n";
3539         # when unplugging usb devices this way,
3540         # there may be remaining usb controllers/hubs
3541         # so we disable it for now
3542         qemu_devicedel($vmid, $deviceid);
3543         qemu_devicedelverify($vmid, $deviceid);
3544
3545     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3546
3547         qemu_devicedel($vmid, $deviceid);
3548         qemu_devicedelverify($vmid, $deviceid);
3549         qemu_drivedel($vmid, $deviceid);
3550         qemu_iothread_del($conf, $vmid, $deviceid);
3551
3552     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3553
3554         qemu_devicedel($vmid, $deviceid);
3555         qemu_devicedelverify($vmid, $deviceid);
3556         qemu_iothread_del($conf, $vmid, $deviceid);
3557
3558     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3559
3560         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3561         my $device = parse_drive($deviceid, $conf->{$deviceid});
3562         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3563
3564         qemu_devicedel($vmid, $deviceid);
3565         qemu_drivedel($vmid, $deviceid);
3566         qemu_deletescsihw($conf, $vmid, $deviceid);
3567
3568     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3569
3570         qemu_devicedel($vmid, $deviceid);
3571         qemu_devicedelverify($vmid, $deviceid);
3572         qemu_netdevdel($vmid, $deviceid);
3573
3574     } else {
3575         die "can't unplug device '$deviceid'\n";
3576     }
3577
3578     return 1;
3579 }
3580
3581 sub qemu_deviceadd {
3582     my ($vmid, $devicefull) = @_;
3583
3584     $devicefull = "driver=".$devicefull;
3585     my %options =  split(/[=,]/, $devicefull);
3586
3587     vm_mon_cmd($vmid, "device_add" , %options);
3588 }
3589
3590 sub qemu_devicedel {
3591     my ($vmid, $deviceid) = @_;
3592
3593     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3594 }
3595
3596 sub qemu_iothread_add {
3597     my($vmid, $deviceid, $device) = @_;
3598
3599     if ($device->{iothread}) {
3600         my $iothreads = vm_iothreads_list($vmid);
3601         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3602     }
3603 }
3604
3605 sub qemu_iothread_del {
3606     my($conf, $vmid, $deviceid) = @_;
3607
3608     my $device = parse_drive($deviceid, $conf->{$deviceid});
3609     if ($device->{iothread}) {
3610         my $iothreads = vm_iothreads_list($vmid);
3611         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3612     }
3613 }
3614
3615 sub qemu_objectadd {
3616     my($vmid, $objectid, $qomtype) = @_;
3617
3618     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3619
3620     return 1;
3621 }
3622
3623 sub qemu_objectdel {
3624     my($vmid, $objectid) = @_;
3625
3626     vm_mon_cmd($vmid, "object-del", id => $objectid);
3627
3628     return 1;
3629 }
3630
3631 sub qemu_driveadd {
3632     my ($storecfg, $vmid, $device) = @_;
3633
3634     my $drive = print_drive_full($storecfg, $vmid, $device);
3635     $drive =~ s/\\/\\\\/g;
3636     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3637
3638     # If the command succeeds qemu prints: "OK"
3639     return 1 if $ret =~ m/OK/s;
3640
3641     die "adding drive failed: $ret\n";
3642 }
3643
3644 sub qemu_drivedel {
3645     my($vmid, $deviceid) = @_;
3646
3647     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3648     $ret =~ s/^\s+//;
3649
3650     return 1 if $ret eq "";
3651
3652     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3653     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3654
3655     die "deleting drive $deviceid failed : $ret\n";
3656 }
3657
3658 sub qemu_deviceaddverify {
3659     my ($vmid, $deviceid) = @_;
3660
3661     for (my $i = 0; $i <= 5; $i++) {
3662          my $devices_list = vm_devices_list($vmid);
3663          return 1 if defined($devices_list->{$deviceid});
3664          sleep 1;
3665     }
3666
3667     die "error on hotplug device '$deviceid'\n";
3668 }
3669
3670
3671 sub qemu_devicedelverify {
3672     my ($vmid, $deviceid) = @_;
3673
3674     # need to verify that the device is correctly removed as device_del
3675     # is async and empty return is not reliable
3676
3677     for (my $i = 0; $i <= 5; $i++) {
3678          my $devices_list = vm_devices_list($vmid);
3679          return 1 if !defined($devices_list->{$deviceid});
3680          sleep 1;
3681     }
3682
3683     die "error on hot-unplugging device '$deviceid'\n";
3684 }
3685
3686 sub qemu_findorcreatescsihw {
3687     my ($storecfg, $conf, $vmid, $device) = @_;
3688
3689     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3690
3691     my $scsihwid="$controller_prefix$controller";
3692     my $devices_list = vm_devices_list($vmid);
3693
3694     if(!defined($devices_list->{$scsihwid})) {
3695         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3696     }
3697
3698     return 1;
3699 }
3700
3701 sub qemu_deletescsihw {
3702     my ($conf, $vmid, $opt) = @_;
3703
3704     my $device = parse_drive($opt, $conf->{$opt});
3705
3706     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3707         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3708         return 1;
3709     }
3710
3711     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3712
3713     my $devices_list = vm_devices_list($vmid);
3714     foreach my $opt (keys %{$devices_list}) {
3715         if (PVE::QemuServer::is_valid_drivename($opt)) {
3716             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3717             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3718                 return 1;
3719             }
3720         }
3721     }
3722
3723     my $scsihwid="scsihw$controller";
3724
3725     vm_deviceunplug($vmid, $conf, $scsihwid);
3726
3727     return 1;
3728 }
3729
3730 sub qemu_add_pci_bridge {
3731     my ($storecfg, $conf, $vmid, $device) = @_;
3732
3733     my $bridges = {};
3734
3735     my $bridgeid;
3736
3737     print_pci_addr($device, $bridges);
3738
3739     while (my ($k, $v) = each %$bridges) {
3740         $bridgeid = $k;
3741     }
3742     return 1 if !defined($bridgeid) || $bridgeid < 1;
3743
3744     my $bridge = "pci.$bridgeid";
3745     my $devices_list = vm_devices_list($vmid);
3746
3747     if (!defined($devices_list->{$bridge})) {
3748         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3749     }
3750
3751     return 1;
3752 }
3753
3754 sub qemu_set_link_status {
3755     my ($vmid, $device, $up) = @_;
3756
3757     vm_mon_cmd($vmid, "set_link", name => $device,
3758                up => $up ? JSON::true : JSON::false);
3759 }
3760
3761 sub qemu_netdevadd {
3762     my ($vmid, $conf, $device, $deviceid) = @_;
3763
3764     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3765     my %options =  split(/[=,]/, $netdev);
3766
3767     vm_mon_cmd($vmid, "netdev_add",  %options);
3768     return 1;
3769 }
3770
3771 sub qemu_netdevdel {
3772     my ($vmid, $deviceid) = @_;
3773
3774     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3775 }
3776
3777 sub qemu_usb_hotplug {
3778     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3779
3780     return if !$device;
3781
3782     # remove the old one first
3783     vm_deviceunplug($vmid, $conf, $deviceid);
3784
3785     # check if xhci controller is necessary and available
3786     if ($device->{usb3}) {
3787
3788         my $devicelist = vm_devices_list($vmid);
3789
3790         if (!$devicelist->{xhci}) {
3791             my $pciaddr = print_pci_addr("xhci");
3792             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3793         }
3794     }
3795     my $d = parse_usb_device($device->{host});
3796     $d->{usb3} = $device->{usb3};
3797
3798     # add the new one
3799     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3800 }
3801
3802 sub qemu_cpu_hotplug {
3803     my ($vmid, $conf, $vcpus) = @_;
3804
3805     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3806
3807     my $sockets = 1;
3808     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3809     $sockets = $conf->{sockets} if  $conf->{sockets};
3810     my $cores = $conf->{cores} || 1;
3811     my $maxcpus = $sockets * $cores;
3812
3813     $vcpus = $maxcpus if !$vcpus;
3814
3815     die "you can't add more vcpus than maxcpus\n"
3816         if $vcpus > $maxcpus;
3817
3818     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3819
3820     if ($vcpus < $currentvcpus) {
3821
3822         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3823
3824             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3825                 qemu_devicedel($vmid, "cpu$i");
3826                 my $retry = 0;
3827                 my $currentrunningvcpus = undef;
3828                 while (1) {
3829                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3830                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3831                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3832                     $retry++;
3833                     sleep 1;
3834                 }
3835                 #update conf after each succesfull cpu unplug
3836                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3837                 PVE::QemuConfig->write_config($vmid, $conf);
3838             }
3839         } else {
3840             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3841         }
3842
3843         return;
3844     }
3845
3846     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3847     die "vcpus in running vm does not match its configuration\n"
3848         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3849
3850     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3851
3852         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3853             my $cpustr = print_cpu_device($conf, $i);
3854             qemu_deviceadd($vmid, $cpustr);
3855
3856             my $retry = 0;
3857             my $currentrunningvcpus = undef;
3858             while (1) {
3859                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3860                 last if scalar(@{$currentrunningvcpus}) == $i;
3861                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3862                 sleep 1;
3863                 $retry++;
3864             }
3865             #update conf after each succesfull cpu hotplug
3866             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3867             PVE::QemuConfig->write_config($vmid, $conf);
3868         }
3869     } else {
3870
3871         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3872             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3873         }
3874     }
3875 }
3876
3877 sub qemu_block_set_io_throttle {
3878     my ($vmid, $deviceid,
3879         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3880         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3881         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3882         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3883
3884     return if !check_running($vmid) ;
3885
3886     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3887         bps => int($bps),
3888         bps_rd => int($bps_rd),
3889         bps_wr => int($bps_wr),
3890         iops => int($iops),
3891         iops_rd => int($iops_rd),
3892         iops_wr => int($iops_wr),
3893         bps_max => int($bps_max),
3894         bps_rd_max => int($bps_rd_max),
3895         bps_wr_max => int($bps_wr_max),
3896         iops_max => int($iops_max),
3897         iops_rd_max => int($iops_rd_max),
3898         iops_wr_max => int($iops_wr_max),
3899         bps_max_length => int($bps_max_length),
3900         bps_rd_max_length => int($bps_rd_max_length),
3901         bps_wr_max_length => int($bps_wr_max_length),
3902         iops_max_length => int($iops_max_length),
3903         iops_rd_max_length => int($iops_rd_max_length),
3904         iops_wr_max_length => int($iops_wr_max_length),
3905     );
3906
3907 }
3908
3909 # old code, only used to shutdown old VM after update
3910 sub __read_avail {
3911     my ($fh, $timeout) = @_;
3912
3913     my $sel = new IO::Select;
3914     $sel->add($fh);
3915
3916     my $res = '';
3917     my $buf;
3918
3919     my @ready;
3920     while (scalar (@ready = $sel->can_read($timeout))) {
3921         my $count;
3922         if ($count = $fh->sysread($buf, 8192)) {
3923             if ($buf =~ /^(.*)\(qemu\) $/s) {
3924                 $res .= $1;
3925                 last;
3926             } else {
3927                 $res .= $buf;
3928             }
3929         } else {
3930             if (!defined($count)) {
3931                 die "$!\n";
3932             }
3933             last;
3934         }
3935     }
3936
3937     die "monitor read timeout\n" if !scalar(@ready);
3938
3939     return $res;
3940 }
3941
3942 # old code, only used to shutdown old VM after update
3943 sub vm_monitor_command {
3944     my ($vmid, $cmdstr, $nocheck) = @_;
3945
3946     my $res;
3947
3948     eval {
3949         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3950
3951         my $sname = "${var_run_tmpdir}/$vmid.mon";
3952
3953         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3954             die "unable to connect to VM $vmid socket - $!\n";
3955
3956         my $timeout = 3;
3957
3958         # hack: migrate sometime blocks the monitor (when migrate_downtime
3959         # is set)
3960         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3961             $timeout = 60*60; # 1 hour
3962         }
3963
3964         # read banner;
3965         my $data = __read_avail($sock, $timeout);
3966
3967         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3968             die "got unexpected qemu monitor banner\n";
3969         }
3970
3971         my $sel = new IO::Select;
3972         $sel->add($sock);
3973
3974         if (!scalar(my @ready = $sel->can_write($timeout))) {
3975             die "monitor write error - timeout";
3976         }
3977
3978         my $fullcmd = "$cmdstr\r";
3979
3980         # syslog('info', "VM $vmid monitor command: $cmdstr");
3981
3982         my $b;
3983         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3984             die "monitor write error - $!";
3985         }
3986
3987         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3988
3989         $timeout = 20;
3990
3991         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3992             $timeout = 60*60; # 1 hour
3993         } elsif ($cmdstr =~ m/^(eject|change)/) {
3994             $timeout = 60; # note: cdrom mount command is slow
3995         }
3996         if ($res = __read_avail($sock, $timeout)) {
3997
3998             my @lines = split("\r?\n", $res);
3999
4000             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4001
4002             $res = join("\n", @lines);
4003             $res .= "\n";
4004         }
4005     };
4006
4007     my $err = $@;
4008
4009     if ($err) {
4010         syslog("err", "VM $vmid monitor command failed - $err");
4011         die $err;
4012     }
4013
4014     return $res;
4015 }
4016
4017 sub qemu_block_resize {
4018     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4019
4020     my $running = check_running($vmid);
4021
4022     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4023
4024     return if !$running;
4025
4026     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4027
4028 }
4029
4030 sub qemu_volume_snapshot {
4031     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4032
4033     my $running = check_running($vmid);
4034
4035     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4036         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4037     } else {
4038         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4039     }
4040 }
4041
4042 sub qemu_volume_snapshot_delete {
4043     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4044
4045     my $running = check_running($vmid);
4046
4047     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4048         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4049     } else {
4050         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4051     }
4052 }
4053
4054 sub set_migration_caps {
4055     my ($vmid) = @_;
4056
4057     my $cap_ref = [];
4058
4059     my $enabled_cap = {
4060         "auto-converge" => 1,
4061         "xbzrle" => 1,
4062         "x-rdma-pin-all" => 0,
4063         "zero-blocks" => 0,
4064         "compress" => 0
4065     };
4066
4067     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4068
4069     for my $supported_capability (@$supported_capabilities) {
4070         push @$cap_ref, {
4071             capability => $supported_capability->{capability},
4072             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4073         };
4074     }
4075
4076     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4077 }
4078
4079 my $fast_plug_option = {
4080     'lock' => 1,
4081     'name' => 1,
4082     'onboot' => 1,
4083     'shares' => 1,
4084     'startup' => 1,
4085     'description' => 1,
4086     'protection' => 1,
4087 };
4088
4089 # hotplug changes in [PENDING]
4090 # $selection hash can be used to only apply specified options, for
4091 # example: { cores => 1 } (only apply changed 'cores')
4092 # $errors ref is used to return error messages
4093 sub vmconfig_hotplug_pending {
4094     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4095
4096     my $defaults = load_defaults();
4097
4098     # commit values which do not have any impact on running VM first
4099     # Note: those option cannot raise errors, we we do not care about
4100     # $selection and always apply them.
4101
4102     my $add_error = sub {
4103         my ($opt, $msg) = @_;
4104         $errors->{$opt} = "hotplug problem - $msg";
4105     };
4106
4107     my $changes = 0;
4108     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4109         if ($fast_plug_option->{$opt}) {
4110             $conf->{$opt} = $conf->{pending}->{$opt};
4111             delete $conf->{pending}->{$opt};
4112             $changes = 1;
4113         }
4114     }
4115
4116     if ($changes) {
4117         PVE::QemuConfig->write_config($vmid, $conf);
4118         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4119     }
4120
4121     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4122
4123     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4124     while (my ($opt, $force) = each %$pending_delete_hash) {
4125         next if $selection && !$selection->{$opt};
4126         eval {
4127             if ($opt eq 'hotplug') {
4128                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4129             } elsif ($opt eq 'tablet') {
4130                 die "skip\n" if !$hotplug_features->{usb};
4131                 if ($defaults->{tablet}) {
4132                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4133                 } else {
4134                     vm_deviceunplug($vmid, $conf, $opt);
4135                 }
4136             } elsif ($opt =~ m/^usb\d+/) {
4137                 die "skip\n";
4138                 # since we cannot reliably hot unplug usb devices
4139                 # we are disabling it
4140                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4141                 vm_deviceunplug($vmid, $conf, $opt);
4142             } elsif ($opt eq 'vcpus') {
4143                 die "skip\n" if !$hotplug_features->{cpu};
4144                 qemu_cpu_hotplug($vmid, $conf, undef);
4145             } elsif ($opt eq 'balloon') {
4146                 # enable balloon device is not hotpluggable
4147                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4148             } elsif ($fast_plug_option->{$opt}) {
4149                 # do nothing
4150             } elsif ($opt =~ m/^net(\d+)$/) {
4151                 die "skip\n" if !$hotplug_features->{network};
4152                 vm_deviceunplug($vmid, $conf, $opt);
4153             } elsif (is_valid_drivename($opt)) {
4154                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4155                 vm_deviceunplug($vmid, $conf, $opt);
4156                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4157             } elsif ($opt =~ m/^memory$/) {
4158                 die "skip\n" if !$hotplug_features->{memory};
4159                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4160             } elsif ($opt eq 'cpuunits') {
4161                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4162             } elsif ($opt eq 'cpulimit') {
4163                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4164             } else {
4165                 die "skip\n";
4166             }
4167         };
4168         if (my $err = $@) {
4169             &$add_error($opt, $err) if $err ne "skip\n";
4170         } else {
4171             # save new config if hotplug was successful
4172             delete $conf->{$opt};
4173             vmconfig_undelete_pending_option($conf, $opt);
4174             PVE::QemuConfig->write_config($vmid, $conf);
4175             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4176         }
4177     }
4178
4179     foreach my $opt (keys %{$conf->{pending}}) {
4180         next if $selection && !$selection->{$opt};
4181         my $value = $conf->{pending}->{$opt};
4182         eval {
4183             if ($opt eq 'hotplug') {
4184                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4185             } elsif ($opt eq 'tablet') {
4186                 die "skip\n" if !$hotplug_features->{usb};
4187                 if ($value == 1) {
4188                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4189                 } elsif ($value == 0) {
4190                     vm_deviceunplug($vmid, $conf, $opt);
4191                 }
4192             } elsif ($opt =~ m/^usb\d+$/) {
4193                 die "skip\n";
4194                 # since we cannot reliably hot unplug usb devices
4195                 # we are disabling it
4196                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4197                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4198                 die "skip\n" if !$d;
4199                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4200             } elsif ($opt eq 'vcpus') {
4201                 die "skip\n" if !$hotplug_features->{cpu};
4202                 qemu_cpu_hotplug($vmid, $conf, $value);
4203             } elsif ($opt eq 'balloon') {
4204                 # enable/disable balloning device is not hotpluggable
4205                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4206                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4207                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4208
4209                 # allow manual ballooning if shares is set to zero
4210                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4211                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4212                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4213                 }
4214             } elsif ($opt =~ m/^net(\d+)$/) {
4215                 # some changes can be done without hotplug
4216                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4217                                     $vmid, $opt, $value);
4218             } elsif (is_valid_drivename($opt)) {
4219                 # some changes can be done without hotplug
4220                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4221                                      $vmid, $opt, $value, 1);
4222             } elsif ($opt =~ m/^memory$/) { #dimms
4223                 die "skip\n" if !$hotplug_features->{memory};
4224                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4225             } elsif ($opt eq 'cpuunits') {
4226                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4227             } elsif ($opt eq 'cpulimit') {
4228                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4229                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4230             } else {
4231                 die "skip\n";  # skip non-hot-pluggable options
4232             }
4233         };
4234         if (my $err = $@) {
4235             &$add_error($opt, $err) if $err ne "skip\n";
4236         } else {
4237             # save new config if hotplug was successful
4238             $conf->{$opt} = $value;
4239             delete $conf->{pending}->{$opt};
4240             PVE::QemuConfig->write_config($vmid, $conf);
4241             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4242         }
4243     }
4244 }
4245
4246 sub try_deallocate_drive {
4247     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4248
4249     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4250         my $volid = $drive->{file};
4251         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4252             my $sid = PVE::Storage::parse_volume_id($volid);
4253             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4254
4255             # check if the disk is really unused
4256             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4257                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4258             PVE::Storage::vdisk_free($storecfg, $volid);
4259             return 1;
4260         } else {
4261             # If vm is not owner of this disk remove from config
4262             return 1;
4263         }
4264     }
4265
4266     return undef;
4267 }
4268
4269 sub vmconfig_delete_or_detach_drive {
4270     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4271
4272     my $drive = parse_drive($opt, $conf->{$opt});
4273
4274     my $rpcenv = PVE::RPCEnvironment::get();
4275     my $authuser = $rpcenv->get_user();
4276
4277     if ($force) {
4278         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4279         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4280     } else {
4281         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4282     }
4283 }
4284
4285 sub vmconfig_apply_pending {
4286     my ($vmid, $conf, $storecfg) = @_;
4287
4288     # cold plug
4289
4290     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4291     while (my ($opt, $force) = each %$pending_delete_hash) {
4292         die "internal error" if $opt =~ m/^unused/;
4293         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4294         if (!defined($conf->{$opt})) {
4295             vmconfig_undelete_pending_option($conf, $opt);
4296             PVE::QemuConfig->write_config($vmid, $conf);
4297         } elsif (is_valid_drivename($opt)) {
4298             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4299             vmconfig_undelete_pending_option($conf, $opt);
4300             delete $conf->{$opt};
4301             PVE::QemuConfig->write_config($vmid, $conf);
4302         } else {
4303             vmconfig_undelete_pending_option($conf, $opt);
4304             delete $conf->{$opt};
4305             PVE::QemuConfig->write_config($vmid, $conf);
4306         }
4307     }
4308
4309     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4310
4311     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4312         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4313
4314         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4315             # skip if nothing changed
4316         } elsif (is_valid_drivename($opt)) {
4317             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4318                 if defined($conf->{$opt});
4319             $conf->{$opt} = $conf->{pending}->{$opt};
4320         } else {
4321             $conf->{$opt} = $conf->{pending}->{$opt};
4322         }
4323
4324         delete $conf->{pending}->{$opt};
4325         PVE::QemuConfig->write_config($vmid, $conf);
4326     }
4327 }
4328
4329 my $safe_num_ne = sub {
4330     my ($a, $b) = @_;
4331
4332     return 0 if !defined($a) && !defined($b);
4333     return 1 if !defined($a);
4334     return 1 if !defined($b);
4335
4336     return $a != $b;
4337 };
4338
4339 my $safe_string_ne = sub {
4340     my ($a, $b) = @_;
4341
4342     return 0 if !defined($a) && !defined($b);
4343     return 1 if !defined($a);
4344     return 1 if !defined($b);
4345
4346     return $a ne $b;
4347 };
4348
4349 sub vmconfig_update_net {
4350     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4351
4352     my $newnet = parse_net($value);
4353
4354     if ($conf->{$opt}) {
4355         my $oldnet = parse_net($conf->{$opt});
4356
4357         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4358             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4359             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4360             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4361
4362             # for non online change, we try to hot-unplug
4363             die "skip\n" if !$hotplug;
4364             vm_deviceunplug($vmid, $conf, $opt);
4365         } else {
4366
4367             die "internal error" if $opt !~ m/net(\d+)/;
4368             my $iface = "tap${vmid}i$1";
4369
4370             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4371                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4372                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4373                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4374                 PVE::Network::tap_unplug($iface);
4375                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4376             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4377                 # Rate can be applied on its own but any change above needs to
4378                 # include the rate in tap_plug since OVS resets everything.
4379                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4380             }
4381
4382             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4383                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4384             }
4385
4386             return 1;
4387         }
4388     }
4389
4390     if ($hotplug) {
4391         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4392     } else {
4393         die "skip\n";
4394     }
4395 }
4396
4397 sub vmconfig_update_disk {
4398     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4399
4400     # fixme: do we need force?
4401
4402     my $drive = parse_drive($opt, $value);
4403
4404     if ($conf->{$opt}) {
4405
4406         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4407
4408             my $media = $drive->{media} || 'disk';
4409             my $oldmedia = $old_drive->{media} || 'disk';
4410             die "unable to change media type\n" if $media ne $oldmedia;
4411
4412             if (!drive_is_cdrom($old_drive)) {
4413
4414                 if ($drive->{file} ne $old_drive->{file}) {
4415
4416                     die "skip\n" if !$hotplug;
4417
4418                     # unplug and register as unused
4419                     vm_deviceunplug($vmid, $conf, $opt);
4420                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4421
4422                 } else {
4423                     # update existing disk
4424
4425                     # skip non hotpluggable value
4426                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4427                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4428                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4429                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4430                         die "skip\n";
4431                     }
4432
4433                     # apply throttle
4434                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4435                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4436                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4437                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4438                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4439                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4440                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4441                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4442                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4443                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4444                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4445                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4446                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4447                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4448                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4449                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4450                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4451                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4452
4453                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4454                                                    ($drive->{mbps} || 0)*1024*1024,
4455                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4456                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4457                                                    $drive->{iops} || 0,
4458                                                    $drive->{iops_rd} || 0,
4459                                                    $drive->{iops_wr} || 0,
4460                                                    ($drive->{mbps_max} || 0)*1024*1024,
4461                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4462                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4463                                                    $drive->{iops_max} || 0,
4464                                                    $drive->{iops_rd_max} || 0,
4465                                                    $drive->{iops_wr_max} || 0,
4466                                                    $drive->{bps_max_length} || 1,
4467                                                    $drive->{bps_rd_max_length} || 1,
4468                                                    $drive->{bps_wr_max_length} || 1,
4469                                                    $drive->{iops_max_length} || 1,
4470                                                    $drive->{iops_rd_max_length} || 1,
4471                                                    $drive->{iops_wr_max_length} || 1);
4472
4473                     }
4474
4475                     return 1;
4476                 }
4477
4478             } else { # cdrom
4479
4480                 if ($drive->{file} eq 'none') {
4481                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4482                 } else {
4483                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4484                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4485                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4486                 }
4487
4488                 return 1;
4489             }
4490         }
4491     }
4492
4493     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4494     # hotplug new disks
4495     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4496     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4497 }
4498
4499 sub vm_start {
4500     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4501         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4502
4503     PVE::QemuConfig->lock_config($vmid, sub {
4504         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4505
4506         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4507
4508         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4509
4510         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4511
4512         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4513             vmconfig_apply_pending($vmid, $conf, $storecfg);
4514             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4515         }
4516
4517         my $defaults = load_defaults();
4518
4519         # set environment variable useful inside network script
4520         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4521
4522         my $local_volumes = {};
4523
4524         if ($targetstorage) {
4525             foreach_drive($conf, sub {
4526                 my ($ds, $drive) = @_;
4527
4528                 return if drive_is_cdrom($drive);
4529
4530                 my $volid = $drive->{file};
4531
4532                 return if !$volid;
4533
4534                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4535
4536                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4537                 return if $scfg->{shared};
4538                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4539             });
4540
4541             my $format = undef;
4542
4543             foreach my $opt (sort keys %$local_volumes) {
4544
4545                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4546                 my $drive = parse_drive($opt, $conf->{$opt});
4547
4548                 #if remote storage is specified, use default format
4549                 if ($targetstorage && $targetstorage ne "1") {
4550                     $storeid = $targetstorage;
4551                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4552                     $format = $defFormat;
4553                 } else {
4554                     #else we use same format than original
4555                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4556                     $format = qemu_img_format($scfg, $volid);
4557                 }
4558
4559                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4560                 my $newdrive = $drive;
4561                 $newdrive->{format} = $format;
4562                 $newdrive->{file} = $newvolid;
4563                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4564                 $local_volumes->{$opt} = $drivestr;
4565                 #pass drive to conf for command line
4566                 $conf->{$opt} = $drivestr;
4567             }
4568         }
4569
4570         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4571
4572         my $migrate_port = 0;
4573         my $migrate_uri;
4574         if ($statefile) {
4575             if ($statefile eq 'tcp') {
4576                 my $localip = "localhost";
4577                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4578                 my $nodename = PVE::INotify::nodename();
4579
4580                 if (!defined($migration_type)) {
4581                     if (defined($datacenterconf->{migration}->{type})) {
4582                         $migration_type = $datacenterconf->{migration}->{type};
4583                     } else {
4584                         $migration_type = 'secure';
4585                     }
4586                 }
4587
4588                 if ($migration_type eq 'insecure') {
4589                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4590                     if ($migrate_network_addr) {
4591                         $localip = $migrate_network_addr;
4592                     } else {
4593                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4594                     }
4595
4596                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4597                 }
4598
4599                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4600                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4601                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4602                 push @$cmd, '-incoming', $migrate_uri;
4603                 push @$cmd, '-S';
4604
4605             } elsif ($statefile eq 'unix') {
4606                 # should be default for secure migrations as a ssh TCP forward
4607                 # tunnel is not deterministic reliable ready and fails regurarly
4608                 # to set up in time, so use UNIX socket forwards
4609                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4610                 unlink $socket_addr;
4611
4612                 $migrate_uri = "unix:$socket_addr";
4613
4614                 push @$cmd, '-incoming', $migrate_uri;
4615                 push @$cmd, '-S';
4616
4617             } else {
4618                 push @$cmd, '-loadstate', $statefile;
4619             }
4620         } elsif ($paused) {
4621             push @$cmd, '-S';
4622         }
4623
4624         # host pci devices
4625         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4626           my $d = parse_hostpci($conf->{"hostpci$i"});
4627           next if !$d;
4628           my $pcidevices = $d->{pciid};
4629           foreach my $pcidevice (@$pcidevices) {
4630                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4631
4632                 my $info = pci_device_info("0000:$pciid");
4633                 die "IOMMU not present\n" if !check_iommu_support();
4634                 die "no pci device info for device '$pciid'\n" if !$info;
4635                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4636                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4637           }
4638         }
4639
4640         PVE::Storage::activate_volumes($storecfg, $vollist);
4641
4642         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4643             my $cmd = [];
4644             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4645             eval  { run_command($cmd); };
4646         }
4647
4648         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4649                                                   : $defaults->{cpuunits};
4650
4651         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4652
4653         my %properties = (
4654             Slice => 'qemu.slice',
4655             KillMode => 'none',
4656             CPUShares => $cpuunits
4657         );
4658
4659         if (my $cpulimit = $conf->{cpulimit}) {
4660             $properties{CPUQuota} = int($cpulimit * 100);
4661         }
4662         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4663
4664         if ($conf->{hugepages}) {
4665
4666             my $code = sub {
4667                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4668                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4669
4670                 PVE::QemuServer::Memory::hugepages_mount();
4671                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4672
4673                 eval  {
4674                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4675                     run_command($cmd, %run_params);
4676                 };
4677
4678                 if (my $err = $@) {
4679                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4680                     die $err;
4681                 }
4682
4683                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4684             };
4685             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4686
4687         } else {
4688             eval  {
4689                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4690                 run_command($cmd, %run_params);
4691             };
4692         }
4693
4694         if (my $err = $@) {
4695             # deactivate volumes if start fails
4696             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4697             die "start failed: $err";
4698         }
4699
4700         print "migration listens on $migrate_uri\n" if $migrate_uri;
4701
4702         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4703             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4704             warn $@ if $@;
4705         }
4706
4707         #start nbd server for storage migration
4708         if ($targetstorage) {
4709             my $nodename = PVE::INotify::nodename();
4710             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4711             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4712             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4713             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4714
4715             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4716
4717             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4718
4719             foreach my $opt (sort keys %$local_volumes) {
4720                 my $volid = $local_volumes->{$opt};
4721                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4722                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4723                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4724             }
4725         }
4726
4727         if ($migratedfrom) {
4728             eval {
4729                 set_migration_caps($vmid);
4730             };
4731             warn $@ if $@;
4732
4733             if ($spice_port) {
4734                 print "spice listens on port $spice_port\n";
4735                 if ($spice_ticket) {
4736                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4737                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4738                 }
4739             }
4740
4741         } else {
4742             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4743                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4744                     if $conf->{balloon};
4745             }
4746
4747             foreach my $opt (keys %$conf) {
4748                 next if $opt !~  m/^net\d+$/;
4749                 my $nicconf = parse_net($conf->{$opt});
4750                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4751             }
4752         }
4753
4754         vm_mon_cmd_nocheck($vmid, 'qom-set',
4755                     path => "machine/peripheral/balloon0",
4756                     property => "guest-stats-polling-interval",
4757                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4758
4759     });
4760 }
4761
4762 sub vm_mon_cmd {
4763     my ($vmid, $execute, %params) = @_;
4764
4765     my $cmd = { execute => $execute, arguments => \%params };
4766     vm_qmp_command($vmid, $cmd);
4767 }
4768
4769 sub vm_mon_cmd_nocheck {
4770     my ($vmid, $execute, %params) = @_;
4771
4772     my $cmd = { execute => $execute, arguments => \%params };
4773     vm_qmp_command($vmid, $cmd, 1);
4774 }
4775
4776 sub vm_qmp_command {
4777     my ($vmid, $cmd, $nocheck) = @_;
4778
4779     my $res;
4780
4781     my $timeout;
4782     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4783         $timeout = $cmd->{arguments}->{timeout};
4784         delete $cmd->{arguments}->{timeout};
4785     }
4786
4787     eval {
4788         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4789         my $sname = qmp_socket($vmid);
4790         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4791             my $qmpclient = PVE::QMPClient->new();
4792
4793             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4794         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4795             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4796                 if scalar(%{$cmd->{arguments}});
4797             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4798         } else {
4799             die "unable to open monitor socket\n";
4800         }
4801     };
4802     if (my $err = $@) {
4803         syslog("err", "VM $vmid qmp command failed - $err");
4804         die $err;
4805     }
4806
4807     return $res;
4808 }
4809
4810 sub vm_human_monitor_command {
4811     my ($vmid, $cmdline) = @_;
4812
4813     my $res;
4814
4815     my $cmd = {
4816         execute => 'human-monitor-command',
4817         arguments => { 'command-line' => $cmdline},
4818     };
4819
4820     return vm_qmp_command($vmid, $cmd);
4821 }
4822
4823 sub vm_commandline {
4824     my ($storecfg, $vmid) = @_;
4825
4826     my $conf = PVE::QemuConfig->load_config($vmid);
4827
4828     my $defaults = load_defaults();
4829
4830     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4831
4832     return PVE::Tools::cmd2string($cmd);
4833 }
4834
4835 sub vm_reset {
4836     my ($vmid, $skiplock) = @_;
4837
4838     PVE::QemuConfig->lock_config($vmid, sub {
4839
4840         my $conf = PVE::QemuConfig->load_config($vmid);
4841
4842         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4843
4844         vm_mon_cmd($vmid, "system_reset");
4845     });
4846 }
4847
4848 sub get_vm_volumes {
4849     my ($conf) = @_;
4850
4851     my $vollist = [];
4852     foreach_volid($conf, sub {
4853         my ($volid, $attr) = @_;
4854
4855         return if $volid =~ m|^/|;
4856
4857         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4858         return if !$sid;
4859
4860         push @$vollist, $volid;
4861     });
4862
4863     return $vollist;
4864 }
4865
4866 sub vm_stop_cleanup {
4867     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4868
4869     eval {
4870
4871         if (!$keepActive) {
4872             my $vollist = get_vm_volumes($conf);
4873             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4874         }
4875
4876         foreach my $ext (qw(mon qmp pid vnc qga)) {
4877             unlink "/var/run/qemu-server/${vmid}.$ext";
4878         }
4879
4880         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4881     };
4882     warn $@ if $@; # avoid errors - just warn
4883 }
4884
4885 # Note: use $nockeck to skip tests if VM configuration file exists.
4886 # We need that when migration VMs to other nodes (files already moved)
4887 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4888 sub vm_stop {
4889     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4890
4891     $force = 1 if !defined($force) && !$shutdown;
4892
4893     if ($migratedfrom){
4894         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4895         kill 15, $pid if $pid;
4896         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4897         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4898         return;
4899     }
4900
4901     PVE::QemuConfig->lock_config($vmid, sub {
4902
4903         my $pid = check_running($vmid, $nocheck);
4904         return if !$pid;
4905
4906         my $conf;
4907         if (!$nocheck) {
4908             $conf = PVE::QemuConfig->load_config($vmid);
4909             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4910             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4911                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4912                 $timeout = $opts->{down} if $opts->{down};
4913             }
4914         }
4915
4916         $timeout = 60 if !defined($timeout);
4917
4918         eval {
4919             if ($shutdown) {
4920                 if (defined($conf) && $conf->{agent}) {
4921                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4922                 } else {
4923                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4924                 }
4925             } else {
4926                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4927             }
4928         };
4929         my $err = $@;
4930
4931         if (!$err) {
4932             my $count = 0;
4933             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4934                 $count++;
4935                 sleep 1;
4936             }
4937
4938             if ($count >= $timeout) {
4939                 if ($force) {
4940                     warn "VM still running - terminating now with SIGTERM\n";
4941                     kill 15, $pid;
4942                 } else {
4943                     die "VM quit/powerdown failed - got timeout\n";
4944                 }
4945             } else {
4946                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4947                 return;
4948             }
4949         } else {
4950             if ($force) {
4951                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4952                 kill 15, $pid;
4953             } else {
4954                 die "VM quit/powerdown failed\n";
4955             }
4956         }
4957
4958         # wait again
4959         $timeout = 10;
4960
4961         my $count = 0;
4962         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4963             $count++;
4964             sleep 1;
4965         }
4966
4967         if ($count >= $timeout) {
4968             warn "VM still running - terminating now with SIGKILL\n";
4969             kill 9, $pid;
4970             sleep 1;
4971         }
4972
4973         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4974    });
4975 }
4976
4977 sub vm_suspend {
4978     my ($vmid, $skiplock) = @_;
4979
4980     PVE::QemuConfig->lock_config($vmid, sub {
4981
4982         my $conf = PVE::QemuConfig->load_config($vmid);
4983
4984         PVE::QemuConfig->check_lock($conf)
4985             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4986
4987         vm_mon_cmd($vmid, "stop");
4988     });
4989 }
4990
4991 sub vm_resume {
4992     my ($vmid, $skiplock, $nocheck) = @_;
4993
4994     PVE::QemuConfig->lock_config($vmid, sub {
4995
4996         if (!$nocheck) {
4997
4998             my $conf = PVE::QemuConfig->load_config($vmid);
4999
5000             PVE::QemuConfig->check_lock($conf)
5001                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5002
5003             vm_mon_cmd($vmid, "cont");
5004
5005         } else {
5006             vm_mon_cmd_nocheck($vmid, "cont");
5007         }
5008     });
5009 }
5010
5011 sub vm_sendkey {
5012     my ($vmid, $skiplock, $key) = @_;
5013
5014     PVE::QemuConfig->lock_config($vmid, sub {
5015
5016         my $conf = PVE::QemuConfig->load_config($vmid);
5017
5018         # there is no qmp command, so we use the human monitor command
5019         vm_human_monitor_command($vmid, "sendkey $key");
5020     });
5021 }
5022
5023 sub vm_destroy {
5024     my ($storecfg, $vmid, $skiplock) = @_;
5025
5026     PVE::QemuConfig->lock_config($vmid, sub {
5027
5028         my $conf = PVE::QemuConfig->load_config($vmid);
5029
5030         if (!check_running($vmid)) {
5031             destroy_vm($storecfg, $vmid, undef, $skiplock);
5032         } else {
5033             die "VM $vmid is running - destroy failed\n";
5034         }
5035     });
5036 }
5037
5038 # pci helpers
5039
5040 sub file_write {
5041     my ($filename, $buf) = @_;
5042
5043     my $fh = IO::File->new($filename, "w");
5044     return undef if !$fh;
5045
5046     my $res = print $fh $buf;
5047
5048     $fh->close();
5049
5050     return $res;
5051 }
5052
5053 sub pci_device_info {
5054     my ($name) = @_;
5055
5056     my $res;
5057
5058     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5059     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5060
5061     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5062     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5063
5064     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5065     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5066
5067     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5068     return undef if !defined($product) || $product !~ s/^0x//;
5069
5070     $res = {
5071         name => $name,
5072         vendor => $vendor,
5073         product => $product,
5074         domain => $domain,
5075         bus => $bus,
5076         slot => $slot,
5077         func => $func,
5078         irq => $irq,
5079         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5080     };
5081
5082     return $res;
5083 }
5084
5085 sub pci_dev_reset {
5086     my ($dev) = @_;
5087
5088     my $name = $dev->{name};
5089
5090     my $fn = "$pcisysfs/devices/$name/reset";
5091
5092     return file_write($fn, "1");
5093 }
5094
5095 sub pci_dev_bind_to_vfio {
5096     my ($dev) = @_;
5097
5098     my $name = $dev->{name};
5099
5100     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5101
5102     if (!-d $vfio_basedir) {
5103         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5104     }
5105     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5106
5107     my $testdir = "$vfio_basedir/$name";
5108     return 1 if -d $testdir;
5109
5110     my $data = "$dev->{vendor} $dev->{product}";
5111     return undef if !file_write("$vfio_basedir/new_id", $data);
5112
5113     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5114     if (!file_write($fn, $name)) {
5115         return undef if -f $fn;
5116     }
5117
5118     $fn = "$vfio_basedir/bind";
5119     if (! -d $testdir) {
5120         return undef if !file_write($fn, $name);
5121     }
5122
5123     return -d $testdir;
5124 }
5125
5126 sub pci_dev_group_bind_to_vfio {
5127     my ($pciid) = @_;
5128
5129     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5130
5131     if (!-d $vfio_basedir) {
5132         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5133     }
5134     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5135
5136     # get IOMMU group devices
5137     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5138       my @devs = grep /^0000:/, readdir($D);
5139     closedir($D);
5140
5141     foreach my $pciid (@devs) {
5142         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5143
5144         # pci bridges, switches or root ports are not supported
5145         # they have a pci_bus subdirectory so skip them
5146         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5147
5148         my $info = pci_device_info($1);
5149         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5150     }
5151
5152     return 1;
5153 }
5154
5155 # vzdump restore implementaion
5156
5157 sub tar_archive_read_firstfile {
5158     my $archive = shift;
5159
5160     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5161
5162     # try to detect archive type first
5163     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5164         die "unable to open file '$archive'\n";
5165     my $firstfile = <$fh>;
5166     kill 15, $pid;
5167     close $fh;
5168
5169     die "ERROR: archive contaions no data\n" if !$firstfile;
5170     chomp $firstfile;
5171
5172     return $firstfile;
5173 }
5174
5175 sub tar_restore_cleanup {
5176     my ($storecfg, $statfile) = @_;
5177
5178     print STDERR "starting cleanup\n";
5179
5180     if (my $fd = IO::File->new($statfile, "r")) {
5181         while (defined(my $line = <$fd>)) {
5182             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5183                 my $volid = $2;
5184                 eval {
5185                     if ($volid =~ m|^/|) {
5186                         unlink $volid || die 'unlink failed\n';
5187                     } else {
5188                         PVE::Storage::vdisk_free($storecfg, $volid);
5189                     }
5190                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5191                 };
5192                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5193             } else {
5194                 print STDERR "unable to parse line in statfile - $line";
5195             }
5196         }
5197         $fd->close();
5198     }
5199 }
5200
5201 sub restore_archive {
5202     my ($archive, $vmid, $user, $opts) = @_;
5203
5204     my $format = $opts->{format};
5205     my $comp;
5206
5207     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5208         $format = 'tar' if !$format;
5209         $comp = 'gzip';
5210     } elsif ($archive =~ m/\.tar$/) {
5211         $format = 'tar' if !$format;
5212     } elsif ($archive =~ m/.tar.lzo$/) {
5213         $format = 'tar' if !$format;
5214         $comp = 'lzop';
5215     } elsif ($archive =~ m/\.vma$/) {
5216         $format = 'vma' if !$format;
5217     } elsif ($archive =~ m/\.vma\.gz$/) {
5218         $format = 'vma' if !$format;
5219         $comp = 'gzip';
5220     } elsif ($archive =~ m/\.vma\.lzo$/) {
5221         $format = 'vma' if !$format;
5222         $comp = 'lzop';
5223     } else {
5224         $format = 'vma' if !$format; # default
5225     }
5226
5227     # try to detect archive format
5228     if ($format eq 'tar') {
5229         return restore_tar_archive($archive, $vmid, $user, $opts);
5230     } else {
5231         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5232     }
5233 }
5234
5235 sub restore_update_config_line {
5236     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5237
5238     return if $line =~ m/^\#qmdump\#/;
5239     return if $line =~ m/^\#vzdump\#/;
5240     return if $line =~ m/^lock:/;
5241     return if $line =~ m/^unused\d+:/;
5242     return if $line =~ m/^parent:/;
5243     return if $line =~ m/^template:/; # restored VM is never a template
5244
5245     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5246     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5247         # try to convert old 1.X settings
5248         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5249         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5250             my ($model, $macaddr) = split(/\=/, $devconfig);
5251             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5252             my $net = {
5253                 model => $model,
5254                 bridge => "vmbr$ind",
5255                 macaddr => $macaddr,
5256             };
5257             my $netstr = print_net($net);
5258
5259             print $outfd "net$cookie->{netcount}: $netstr\n";
5260             $cookie->{netcount}++;
5261         }
5262     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5263         my ($id, $netstr) = ($1, $2);
5264         my $net = parse_net($netstr);
5265         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5266         $netstr = print_net($net);
5267         print $outfd "$id: $netstr\n";
5268     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5269         my $virtdev = $1;
5270         my $value = $3;
5271         my $di = parse_drive($virtdev, $value);
5272         if (defined($di->{backup}) && !$di->{backup}) {
5273             print $outfd "#$line";
5274         } elsif ($map->{$virtdev}) {
5275             delete $di->{format}; # format can change on restore
5276             $di->{file} = $map->{$virtdev};
5277             $value = print_drive($vmid, $di);
5278             print $outfd "$virtdev: $value\n";
5279         } else {
5280             print $outfd $line;
5281         }
5282     } else {
5283         print $outfd $line;
5284     }
5285 }
5286
5287 sub scan_volids {
5288     my ($cfg, $vmid) = @_;
5289
5290     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5291
5292     my $volid_hash = {};
5293     foreach my $storeid (keys %$info) {
5294         foreach my $item (@{$info->{$storeid}}) {
5295             next if !($item->{volid} && $item->{size});
5296             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5297             $volid_hash->{$item->{volid}} = $item;
5298         }
5299     }
5300
5301     return $volid_hash;
5302 }
5303
5304 sub is_volume_in_use {
5305     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5306
5307     my $path = PVE::Storage::path($storecfg, $volid);
5308
5309     my $scan_config = sub {
5310         my ($cref, $snapname) = @_;
5311
5312         foreach my $key (keys %$cref) {
5313             my $value = $cref->{$key};
5314             if (is_valid_drivename($key)) {
5315                 next if $skip_drive && $key eq $skip_drive;
5316                 my $drive = parse_drive($key, $value);
5317                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5318                 return 1 if $volid eq $drive->{file};
5319                 if ($drive->{file} =~ m!^/!) {
5320                     return 1 if $drive->{file} eq $path;
5321                 } else {
5322                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5323                     next if !$storeid;
5324                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5325                     next if !$scfg;
5326                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5327                 }
5328             }
5329         }
5330
5331         return 0;
5332     };
5333
5334     return 1 if &$scan_config($conf);
5335
5336     undef $skip_drive;
5337
5338     foreach my $snapname (keys %{$conf->{snapshots}}) {
5339         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5340     }
5341
5342     return 0;
5343 }
5344
5345 sub update_disksize {
5346     my ($vmid, $conf, $volid_hash) = @_;
5347
5348     my $changes;
5349
5350     my $used = {};
5351
5352     # Note: it is allowed to define multiple storages with same path (alias), so
5353     # we need to check both 'volid' and real 'path' (two different volid can point
5354     # to the same path).
5355
5356     my $usedpath = {};
5357
5358     # update size info
5359     foreach my $opt (keys %$conf) {
5360         if (is_valid_drivename($opt)) {
5361             my $drive = parse_drive($opt, $conf->{$opt});
5362             my $volid = $drive->{file};
5363             next if !$volid;
5364
5365             $used->{$volid} = 1;
5366             if ($volid_hash->{$volid} &&
5367                 (my $path = $volid_hash->{$volid}->{path})) {
5368                 $usedpath->{$path} = 1;
5369             }
5370
5371             next if drive_is_cdrom($drive);
5372             next if !$volid_hash->{$volid};
5373
5374             $drive->{size} = $volid_hash->{$volid}->{size};
5375             my $new = print_drive($vmid, $drive);
5376             if ($new ne $conf->{$opt}) {
5377                 $changes = 1;
5378                 $conf->{$opt} = $new;
5379             }
5380         }
5381     }
5382
5383     # remove 'unusedX' entry if volume is used
5384     foreach my $opt (keys %$conf) {
5385         next if $opt !~ m/^unused\d+$/;
5386         my $volid = $conf->{$opt};
5387         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5388         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5389             $changes = 1;
5390             delete $conf->{$opt};
5391         }
5392     }
5393
5394     foreach my $volid (sort keys %$volid_hash) {
5395         next if $volid =~ m/vm-$vmid-state-/;
5396         next if $used->{$volid};
5397         my $path = $volid_hash->{$volid}->{path};
5398         next if !$path; # just to be sure
5399         next if $usedpath->{$path};
5400         $changes = 1;
5401         PVE::QemuConfig->add_unused_volume($conf, $volid);
5402         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5403     }
5404
5405     return $changes;
5406 }
5407
5408 sub rescan {
5409     my ($vmid, $nolock) = @_;
5410
5411     my $cfg = PVE::Storage::config();
5412
5413     my $volid_hash = scan_volids($cfg, $vmid);
5414
5415     my $updatefn =  sub {
5416         my ($vmid) = @_;
5417
5418         my $conf = PVE::QemuConfig->load_config($vmid);
5419
5420         PVE::QemuConfig->check_lock($conf);
5421
5422         my $vm_volids = {};
5423         foreach my $volid (keys %$volid_hash) {
5424             my $info = $volid_hash->{$volid};
5425             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5426         }
5427
5428         my $changes = update_disksize($vmid, $conf, $vm_volids);
5429
5430         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5431     };
5432
5433     if (defined($vmid)) {
5434         if ($nolock) {
5435             &$updatefn($vmid);
5436         } else {
5437             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5438         }
5439     } else {
5440         my $vmlist = config_list();
5441         foreach my $vmid (keys %$vmlist) {
5442             if ($nolock) {
5443                 &$updatefn($vmid);
5444             } else {
5445                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5446             }
5447         }
5448     }
5449 }
5450
5451 sub restore_vma_archive {
5452     my ($archive, $vmid, $user, $opts, $comp) = @_;
5453
5454     my $input = $archive eq '-' ? "<&STDIN" : undef;
5455     my $readfrom = $archive;
5456
5457     my $uncomp = '';
5458     if ($comp) {
5459         $readfrom = '-';
5460         my $qarchive = PVE::Tools::shellquote($archive);
5461         if ($comp eq 'gzip') {
5462             $uncomp = "zcat $qarchive|";
5463         } elsif ($comp eq 'lzop') {
5464             $uncomp = "lzop -d -c $qarchive|";
5465         } else {
5466             die "unknown compression method '$comp'\n";
5467         }
5468
5469     }
5470
5471     my $tmpdir = "/var/tmp/vzdumptmp$$";
5472     rmtree $tmpdir;
5473
5474     # disable interrupts (always do cleanups)
5475     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5476         warn "got interrupt - ignored\n";
5477     };
5478
5479     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5480     POSIX::mkfifo($mapfifo, 0600);
5481     my $fifofh;
5482
5483     my $openfifo = sub {
5484         open($fifofh, '>', $mapfifo) || die $!;
5485     };
5486
5487     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5488
5489     my $oldtimeout;
5490     my $timeout = 5;
5491
5492     my $devinfo = {};
5493
5494     my $rpcenv = PVE::RPCEnvironment::get();
5495
5496     my $conffile = PVE::QemuConfig->config_file($vmid);
5497     my $tmpfn = "$conffile.$$.tmp";
5498
5499     # Note: $oldconf is undef if VM does not exists
5500     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5501     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5502
5503     my $print_devmap = sub {
5504         my $virtdev_hash = {};
5505
5506         my $cfgfn = "$tmpdir/qemu-server.conf";
5507
5508         # we can read the config - that is already extracted
5509         my $fh = IO::File->new($cfgfn, "r") ||
5510             "unable to read qemu-server.conf - $!\n";
5511
5512         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5513         if (-f $fwcfgfn) {
5514             my $pve_firewall_dir = '/etc/pve/firewall';
5515             mkdir $pve_firewall_dir; # make sure the dir exists
5516             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5517         }
5518
5519         while (defined(my $line = <$fh>)) {
5520             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5521                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5522                 die "archive does not contain data for drive '$virtdev'\n"
5523                     if !$devinfo->{$devname};
5524                 if (defined($opts->{storage})) {
5525                     $storeid = $opts->{storage} || 'local';
5526                 } elsif (!$storeid) {
5527                     $storeid = 'local';
5528                 }
5529                 $format = 'raw' if !$format;
5530                 $devinfo->{$devname}->{devname} = $devname;
5531                 $devinfo->{$devname}->{virtdev} = $virtdev;
5532                 $devinfo->{$devname}->{format} = $format;
5533                 $devinfo->{$devname}->{storeid} = $storeid;
5534
5535                 # check permission on storage
5536                 my $pool = $opts->{pool}; # todo: do we need that?
5537                 if ($user ne 'root@pam') {
5538                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5539                 }
5540
5541                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5542             }
5543         }
5544
5545         foreach my $devname (keys %$devinfo) {
5546             die "found no device mapping information for device '$devname'\n"
5547                 if !$devinfo->{$devname}->{virtdev};
5548         }
5549
5550         my $cfg = PVE::Storage::config();
5551
5552         # create empty/temp config
5553         if ($oldconf) {
5554             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5555             foreach_drive($oldconf, sub {
5556                 my ($ds, $drive) = @_;
5557
5558                 return if drive_is_cdrom($drive);
5559
5560                 my $volid = $drive->{file};
5561
5562                 return if !$volid || $volid =~ m|^/|;
5563
5564                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5565                 return if !$path || !$owner || ($owner != $vmid);
5566
5567                 # Note: only delete disk we want to restore
5568                 # other volumes will become unused
5569                 if ($virtdev_hash->{$ds}) {
5570                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5571                     if (my $err = $@) {
5572                         warn $err;
5573                     }
5574                 }
5575             });
5576
5577             # delete vmstate files
5578             # since after the restore we have no snapshots anymore
5579             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5580                 my $snap = $oldconf->{snapshots}->{$snapname};
5581                 if ($snap->{vmstate}) {
5582                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5583                     if (my $err = $@) {
5584                         warn $err;
5585                     }
5586                 }
5587             }
5588         }
5589
5590         my $map = {};
5591         foreach my $virtdev (sort keys %$virtdev_hash) {
5592             my $d = $virtdev_hash->{$virtdev};
5593             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5594             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5595
5596             # test if requested format is supported
5597             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5598             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5599             $d->{format} = $defFormat if !$supported;
5600
5601             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5602                                                   $d->{format}, undef, $alloc_size);
5603             print STDERR "new volume ID is '$volid'\n";
5604             $d->{volid} = $volid;
5605             my $path = PVE::Storage::path($cfg, $volid);
5606
5607             PVE::Storage::activate_volumes($cfg,[$volid]);
5608
5609             my $write_zeros = 1;
5610             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5611                 $write_zeros = 0;
5612             }
5613
5614             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5615
5616             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5617             $map->{$virtdev} = $volid;
5618         }
5619
5620         $fh->seek(0, 0) || die "seek failed - $!\n";
5621
5622         my $outfd = new IO::File ($tmpfn, "w") ||
5623             die "unable to write config for VM $vmid\n";
5624
5625         my $cookie = { netcount => 0 };
5626         while (defined(my $line = <$fh>)) {
5627             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5628         }
5629
5630         $fh->close();
5631         $outfd->close();
5632     };
5633
5634     eval {
5635         # enable interrupts
5636         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5637             die "interrupted by signal\n";
5638         };
5639         local $SIG{ALRM} = sub { die "got timeout\n"; };
5640
5641         $oldtimeout = alarm($timeout);
5642
5643         my $parser = sub {
5644             my $line = shift;
5645
5646             print "$line\n";
5647
5648             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5649                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5650                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5651             } elsif ($line =~ m/^CTIME: /) {
5652                 # we correctly received the vma config, so we can disable
5653                 # the timeout now for disk allocation (set to 10 minutes, so
5654                 # that we always timeout if something goes wrong)
5655                 alarm(600);
5656                 &$print_devmap();
5657                 print $fifofh "done\n";
5658                 my $tmp = $oldtimeout || 0;
5659                 $oldtimeout = undef;
5660                 alarm($tmp);
5661                 close($fifofh);
5662             }
5663         };
5664
5665         print "restore vma archive: $cmd\n";
5666         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5667     };
5668     my $err = $@;
5669
5670     alarm($oldtimeout) if $oldtimeout;
5671
5672     my $vollist = [];
5673     foreach my $devname (keys %$devinfo) {
5674         my $volid = $devinfo->{$devname}->{volid};
5675         push @$vollist, $volid if $volid;
5676     }
5677
5678     my $cfg = PVE::Storage::config();
5679     PVE::Storage::deactivate_volumes($cfg, $vollist);
5680
5681     unlink $mapfifo;
5682
5683     if ($err) {
5684         rmtree $tmpdir;
5685         unlink $tmpfn;
5686
5687         foreach my $devname (keys %$devinfo) {
5688             my $volid = $devinfo->{$devname}->{volid};
5689             next if !$volid;
5690             eval {
5691                 if ($volid =~ m|^/|) {
5692                     unlink $volid || die 'unlink failed\n';
5693                 } else {
5694                     PVE::Storage::vdisk_free($cfg, $volid);
5695                 }
5696                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5697             };
5698             print STDERR "unable to cleanup '$volid' - $@" if $@;
5699         }
5700         die $err;
5701     }
5702
5703     rmtree $tmpdir;
5704
5705     rename($tmpfn, $conffile) ||
5706         die "unable to commit configuration file '$conffile'\n";
5707
5708     PVE::Cluster::cfs_update(); # make sure we read new file
5709
5710     eval { rescan($vmid, 1); };
5711     warn $@ if $@;
5712 }
5713
5714 sub restore_tar_archive {
5715     my ($archive, $vmid, $user, $opts) = @_;
5716
5717     if ($archive ne '-') {
5718         my $firstfile = tar_archive_read_firstfile($archive);
5719         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5720             if $firstfile ne 'qemu-server.conf';
5721     }
5722
5723     my $storecfg = PVE::Storage::config();
5724
5725     # destroy existing data - keep empty config
5726     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5727     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5728
5729     my $tocmd = "/usr/lib/qemu-server/qmextract";
5730
5731     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5732     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5733     $tocmd .= ' --prealloc' if $opts->{prealloc};
5734     $tocmd .= ' --info' if $opts->{info};
5735
5736     # tar option "xf" does not autodetect compression when read from STDIN,
5737     # so we pipe to zcat
5738     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5739         PVE::Tools::shellquote("--to-command=$tocmd");
5740
5741     my $tmpdir = "/var/tmp/vzdumptmp$$";
5742     mkpath $tmpdir;
5743
5744     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5745     local $ENV{VZDUMP_VMID} = $vmid;
5746     local $ENV{VZDUMP_USER} = $user;
5747
5748     my $conffile = PVE::QemuConfig->config_file($vmid);
5749     my $tmpfn = "$conffile.$$.tmp";
5750
5751     # disable interrupts (always do cleanups)
5752     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5753         print STDERR "got interrupt - ignored\n";
5754     };
5755
5756     eval {
5757         # enable interrupts
5758         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5759             die "interrupted by signal\n";
5760         };
5761
5762         if ($archive eq '-') {
5763             print "extracting archive from STDIN\n";
5764             run_command($cmd, input => "<&STDIN");
5765         } else {
5766             print "extracting archive '$archive'\n";
5767             run_command($cmd);
5768         }
5769
5770         return if $opts->{info};
5771
5772         # read new mapping
5773         my $map = {};
5774         my $statfile = "$tmpdir/qmrestore.stat";
5775         if (my $fd = IO::File->new($statfile, "r")) {
5776             while (defined (my $line = <$fd>)) {
5777                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5778                     $map->{$1} = $2 if $1;
5779                 } else {
5780                     print STDERR "unable to parse line in statfile - $line\n";
5781                 }
5782             }
5783             $fd->close();
5784         }
5785
5786         my $confsrc = "$tmpdir/qemu-server.conf";
5787
5788         my $srcfd = new IO::File($confsrc, "r") ||
5789             die "unable to open file '$confsrc'\n";
5790
5791         my $outfd = new IO::File ($tmpfn, "w") ||
5792             die "unable to write config for VM $vmid\n";
5793
5794         my $cookie = { netcount => 0 };
5795         while (defined (my $line = <$srcfd>)) {
5796             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5797         }
5798
5799         $srcfd->close();
5800         $outfd->close();
5801     };
5802     my $err = $@;
5803
5804     if ($err) {
5805
5806         unlink $tmpfn;
5807
5808         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5809
5810         die $err;
5811     }
5812
5813     rmtree $tmpdir;
5814
5815     rename $tmpfn, $conffile ||
5816         die "unable to commit configuration file '$conffile'\n";
5817
5818     PVE::Cluster::cfs_update(); # make sure we read new file
5819
5820     eval { rescan($vmid, 1); };
5821     warn $@ if $@;
5822 };
5823
5824 sub foreach_writable_storage {
5825     my ($conf, $func) = @_;
5826
5827     my $sidhash = {};
5828
5829     foreach my $ds (keys %$conf) {
5830         next if !is_valid_drivename($ds);
5831
5832         my $drive = parse_drive($ds, $conf->{$ds});
5833         next if !$drive;
5834         next if drive_is_cdrom($drive);
5835
5836         my $volid = $drive->{file};
5837
5838         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5839         $sidhash->{$sid} = $sid if $sid;
5840     }
5841
5842     foreach my $sid (sort keys %$sidhash) {
5843         &$func($sid);
5844     }
5845 }
5846
5847 sub do_snapshots_with_qemu {
5848     my ($storecfg, $volid) = @_;
5849
5850     my $storage_name = PVE::Storage::parse_volume_id($volid);
5851
5852     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5853         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5854         return 1;
5855     }
5856
5857     if ($volid =~ m/\.(qcow2|qed)$/){
5858         return 1;
5859     }
5860
5861     return undef;
5862 }
5863
5864 sub qga_check_running {
5865     my ($vmid) = @_;
5866
5867     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5868     if ($@) {
5869         warn "Qemu Guest Agent are not running - $@";
5870         return 0;
5871     }
5872     return 1;
5873 }
5874
5875 sub template_create {
5876     my ($vmid, $conf, $disk) = @_;
5877
5878     my $storecfg = PVE::Storage::config();
5879
5880     foreach_drive($conf, sub {
5881         my ($ds, $drive) = @_;
5882
5883         return if drive_is_cdrom($drive);
5884         return if $disk && $ds ne $disk;
5885
5886         my $volid = $drive->{file};
5887         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5888
5889         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5890         $drive->{file} = $voliddst;
5891         $conf->{$ds} = print_drive($vmid, $drive);
5892         PVE::QemuConfig->write_config($vmid, $conf);
5893     });
5894 }
5895
5896 sub qemu_img_convert {
5897     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5898
5899     my $storecfg = PVE::Storage::config();
5900     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5901     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5902
5903     if ($src_storeid && $dst_storeid) {
5904
5905         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5906
5907         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5908         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5909
5910         my $src_format = qemu_img_format($src_scfg, $src_volname);
5911         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5912
5913         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5914         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5915
5916         my $cmd = [];
5917         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5918         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5919         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5920         if ($is_zero_initialized) {
5921             push @$cmd, "zeroinit:$dst_path";
5922         } else {
5923             push @$cmd, $dst_path;
5924         }
5925
5926         my $parser = sub {
5927             my $line = shift;
5928             if($line =~ m/\((\S+)\/100\%\)/){
5929                 my $percent = $1;
5930                 my $transferred = int($size * $percent / 100);
5931                 my $remaining = $size - $transferred;
5932
5933                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5934             }
5935
5936         };
5937
5938         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5939         my $err = $@;
5940         die "copy failed: $err" if $err;
5941     }
5942 }
5943
5944 sub qemu_img_format {
5945     my ($scfg, $volname) = @_;
5946
5947     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5948         return $1;
5949     } else {
5950         return "raw";
5951     }
5952 }
5953
5954 sub qemu_drive_mirror {
5955     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5956
5957     $jobs = {} if !$jobs;
5958
5959     my $qemu_target;
5960     my $format;
5961     $jobs->{"drive-$drive"} = {};
5962
5963     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5964         my $server = $1;
5965         my $port = $2;
5966         my $exportname = $3;
5967
5968         $format = "nbd";
5969         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5970         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5971         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5972
5973         my $pid = fork();
5974         if (!defined($pid)) {
5975             die "forking socat tunnel failed\n";
5976         } elsif ($pid == 0) {
5977             exec(@$cmd);
5978             warn "exec failed: $!\n";
5979             POSIX::_exit(-1);
5980         }
5981         $jobs->{"drive-$drive"}->{pid} = $pid;
5982
5983         my $timeout = 0;
5984         while (!-S $unixsocket) {
5985             die "nbd connection helper timed out\n"
5986                 if $timeout++ > 5;
5987             sleep 1;
5988         }
5989     } else {
5990         my $storecfg = PVE::Storage::config();
5991         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5992
5993         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5994
5995         $format = qemu_img_format($dst_scfg, $dst_volname);
5996
5997         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5998
5999         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6000     }
6001
6002     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6003     $opts->{format} = $format if $format;
6004
6005     print "drive mirror is starting for drive-$drive\n";
6006
6007     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6008
6009     if (my $err = $@) {
6010         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6011         die "mirroring error: $err";
6012     }
6013
6014     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6015 }
6016
6017 sub qemu_drive_mirror_monitor {
6018     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6019
6020     eval {
6021         my $err_complete = 0;
6022
6023         while (1) {
6024             die "storage migration timed out\n" if $err_complete > 300;
6025
6026             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6027
6028             my $running_mirror_jobs = {};
6029             foreach my $stat (@$stats) {
6030                 next if $stat->{type} ne 'mirror';
6031                 $running_mirror_jobs->{$stat->{device}} = $stat;
6032             }
6033
6034             my $readycounter = 0;
6035
6036             foreach my $job (keys %$jobs) {
6037
6038                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6039                     print "$job : finished\n";
6040                     delete $jobs->{$job};
6041                     next;
6042                 }
6043
6044                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6045
6046                 my $busy = $running_mirror_jobs->{$job}->{busy};
6047                 my $ready = $running_mirror_jobs->{$job}->{ready};
6048                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6049                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6050                     my $remaining = $total - $transferred;
6051                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6052
6053                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6054                 }
6055
6056                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6057             }
6058
6059             last if scalar(keys %$jobs) == 0;
6060
6061             if ($readycounter == scalar(keys %$jobs)) {
6062                 print "all mirroring jobs are ready \n";
6063                 last if $skipcomplete; #do the complete later
6064
6065                 if ($vmiddst && $vmiddst != $vmid) {
6066                     if ($qga) {
6067                         print "freeze filesystem\n";
6068                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6069                     } else {
6070                         print "suspend vm\n";
6071                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6072                     }
6073
6074                     # if we clone a disk for a new target vm, we don't switch the disk
6075                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6076
6077                     if ($qga) {
6078                         print "unfreeze filesystem\n";
6079                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6080                     } else {
6081                         print "resume vm\n";
6082                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6083                     }
6084
6085                     last;
6086                 } else {
6087
6088                     foreach my $job (keys %$jobs) {
6089                         # try to switch the disk if source and destination are on the same guest
6090                         print "$job: Completing block job...\n";
6091
6092                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6093                         if ($@ =~ m/cannot be completed/) {
6094                             print "$job: Block job cannot be completed, try again.\n";
6095                             $err_complete++;
6096                         }else {
6097                             print "$job: Completed successfully.\n";
6098                             $jobs->{$job}->{complete} = 1;
6099                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6100                         }
6101                     }
6102                 }
6103             }
6104             sleep 1;
6105         }
6106     };
6107     my $err = $@;
6108
6109     if ($err) {
6110         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6111         die "mirroring error: $err";
6112     }
6113
6114 }
6115
6116 sub qemu_blockjobs_cancel {
6117     my ($vmid, $jobs) = @_;
6118
6119     foreach my $job (keys %$jobs) {
6120         print "$job: Cancelling block job\n";
6121         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6122         $jobs->{$job}->{cancel} = 1;
6123     }
6124
6125     while (1) {
6126         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6127
6128         my $running_jobs = {};
6129         foreach my $stat (@$stats) {
6130             $running_jobs->{$stat->{device}} = $stat;
6131         }
6132
6133         foreach my $job (keys %$jobs) {
6134
6135             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6136                 print "$job: Done.\n";
6137                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6138                 delete $jobs->{$job};
6139             }
6140         }
6141
6142         last if scalar(keys %$jobs) == 0;
6143
6144         sleep 1;
6145     }
6146 }
6147
6148 sub qemu_blockjobs_finish_tunnel {
6149    my ($vmid, $job, $cpid) = @_;
6150
6151    return if !$cpid;
6152
6153    for (my $i = 1; $i < 20; $i++) {
6154         my $waitpid = waitpid($cpid, WNOHANG);
6155         last if (defined($waitpid) && ($waitpid == $cpid));
6156
6157         if ($i == 10) {
6158             kill(15, $cpid);
6159         } elsif ($i >= 15) {
6160             kill(9, $cpid);
6161         }
6162         sleep (1);
6163     }
6164     unlink "/run/qemu-server/$vmid.mirror-$job";
6165 }
6166
6167 sub clone_disk {
6168     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6169         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6170
6171     my $newvolid;
6172
6173     if (!$full) {
6174         print "create linked clone of drive $drivename ($drive->{file})\n";
6175         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6176         push @$newvollist, $newvolid;
6177     } else {
6178
6179         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6180         $storeid = $storage if $storage;
6181
6182         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6183         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6184
6185         print "create full clone of drive $drivename ($drive->{file})\n";
6186         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6187         push @$newvollist, $newvolid;
6188
6189         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6190
6191         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6192         if (!$running || $snapname) {
6193             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6194         } else {
6195
6196             my $kvmver = get_running_qemu_version ($vmid);
6197             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6198                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6199                     if $drive->{iothread};
6200             }
6201
6202             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6203         }
6204     }
6205
6206     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6207
6208     my $disk = $drive;
6209     $disk->{format} = undef;
6210     $disk->{file} = $newvolid;
6211     $disk->{size} = $size;
6212
6213     return $disk;
6214 }
6215
6216 # this only works if VM is running
6217 sub get_current_qemu_machine {
6218     my ($vmid) = @_;
6219
6220     my $cmd = { execute => 'query-machines', arguments => {} };
6221     my $res = vm_qmp_command($vmid, $cmd);
6222
6223     my ($current, $default);
6224     foreach my $e (@$res) {
6225         $default = $e->{name} if $e->{'is-default'};
6226         $current = $e->{name} if $e->{'is-current'};
6227     }
6228
6229     # fallback to the default machine if current is not supported by qemu
6230     return $current || $default || 'pc';
6231 }
6232
6233 sub get_running_qemu_version {
6234     my ($vmid) = @_;
6235     my $cmd = { execute => 'query-version', arguments => {} };
6236     my $res = vm_qmp_command($vmid, $cmd);
6237     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6238 }
6239
6240 sub qemu_machine_feature_enabled {
6241     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6242
6243     my $current_major;
6244     my $current_minor;
6245
6246     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6247
6248         $current_major = $3;
6249         $current_minor = $4;
6250
6251     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6252
6253         $current_major = $1;
6254         $current_minor = $2;
6255     }
6256
6257     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6258
6259
6260 }
6261
6262 sub qemu_machine_pxe {
6263     my ($vmid, $conf, $machine) = @_;
6264
6265     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6266
6267     foreach my $opt (keys %$conf) {
6268         next if $opt !~ m/^net(\d+)$/;
6269         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6270         next if !$net;
6271         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6272         return $machine.".pxe" if $romfile =~ m/pxe/;
6273         last;
6274     }
6275
6276     return $machine;
6277 }
6278
6279 sub qemu_use_old_bios_files {
6280     my ($machine_type) = @_;
6281
6282     return if !$machine_type;
6283
6284     my $use_old_bios_files = undef;
6285
6286     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6287         $machine_type = $1;
6288         $use_old_bios_files = 1;
6289     } else {
6290         my $kvmver = kvm_user_version();
6291         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6292         # load new efi bios files on migration. So this hack is required to allow
6293         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6294         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6295         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6296     }
6297
6298     return ($use_old_bios_files, $machine_type);
6299 }
6300
6301 sub lspci {
6302
6303     my $devices = {};
6304
6305     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6306             my (undef, $id, $function) = @_;
6307             my $res = { id => $id, function => $function};
6308             push @{$devices->{$id}}, $res;
6309     });
6310
6311     # Entries should be sorted by functions.
6312     foreach my $id (keys %$devices) {
6313         my $dev = $devices->{$id};
6314         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6315     }
6316
6317     return $devices;
6318 }
6319
6320 sub vm_iothreads_list {
6321     my ($vmid) = @_;
6322
6323     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6324
6325     my $iothreads = {};
6326     foreach my $iothread (@$res) {
6327         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6328     }
6329
6330     return $iothreads;
6331 }
6332
6333 sub scsihw_infos {
6334     my ($conf, $drive) = @_;
6335
6336     my $maxdev = 0;
6337
6338     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6339         $maxdev = 7;
6340     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6341         $maxdev = 1;
6342     } else {
6343         $maxdev = 256;
6344     }
6345
6346     my $controller = int($drive->{index} / $maxdev);
6347     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6348
6349     return ($maxdev, $controller, $controller_prefix);
6350 }
6351
6352 sub add_hyperv_enlighments {
6353     my ($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $bios, $gpu_passthrough) = @_;
6354
6355     return if $nokvm;
6356     return if $winversion < 6;
6357     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6358
6359     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6360
6361     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6362         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6363         push @$cpuFlags , 'hv_vapic';
6364         push @$cpuFlags , 'hv_time';
6365     } else {
6366         push @$cpuFlags , 'hv_spinlocks=0xffff';
6367     }
6368
6369     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6370         push @$cpuFlags , 'hv_reset';
6371         push @$cpuFlags , 'hv_vpindex';
6372         push @$cpuFlags , 'hv_runtime';
6373     }
6374
6375     if ($winversion >= 7) {
6376         push @$cpuFlags , 'hv_relaxed';
6377     }
6378 }
6379
6380 sub windows_version {
6381     my ($ostype) = @_;
6382
6383     return 0 if !$ostype;
6384
6385     my $winversion = 0;
6386
6387     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6388         $winversion = 5;
6389     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6390         $winversion = 6;
6391     } elsif ($ostype =~ m/^win(\d+)$/) {
6392         $winversion = $1;
6393     }
6394
6395     return $winversion;
6396 }
6397
6398 sub resolve_dst_disk_format {
6399         my ($storecfg, $storeid, $src_volname, $format) = @_;
6400         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6401
6402         if (!$format) {
6403             # if no target format is specified, use the source disk format as hint
6404             if ($src_volname) {
6405                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6406                 $format = qemu_img_format($scfg, $src_volname);
6407             } else {
6408                 return $defFormat;
6409             }
6410         }
6411
6412         # test if requested format is supported - else use default
6413         my $supported = grep { $_ eq $format } @$validFormats;
6414         $format = $defFormat if !$supported;
6415         return $format;
6416 }
6417
6418 # bash completion helper
6419
6420 sub complete_backup_archives {
6421     my ($cmdname, $pname, $cvalue) = @_;
6422
6423     my $cfg = PVE::Storage::config();
6424
6425     my $storeid;
6426
6427     if ($cvalue =~ m/^([^:]+):/) {
6428         $storeid = $1;
6429     }
6430
6431     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6432
6433     my $res = [];
6434     foreach my $id (keys %$data) {
6435         foreach my $item (@{$data->{$id}}) {
6436             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6437             push @$res, $item->{volid} if defined($item->{volid});
6438         }
6439     }
6440
6441     return $res;
6442 }
6443
6444 my $complete_vmid_full = sub {
6445     my ($running) = @_;
6446
6447     my $idlist = vmstatus();
6448
6449     my $res = [];
6450
6451     foreach my $id (keys %$idlist) {
6452         my $d = $idlist->{$id};
6453         if (defined($running)) {
6454             next if $d->{template};
6455             next if $running && $d->{status} ne 'running';
6456             next if !$running && $d->{status} eq 'running';
6457         }
6458         push @$res, $id;
6459
6460     }
6461     return $res;
6462 };
6463
6464 sub complete_vmid {
6465     return &$complete_vmid_full();
6466 }
6467
6468 sub complete_vmid_stopped {
6469     return &$complete_vmid_full(0);
6470 }
6471
6472 sub complete_vmid_running {
6473     return &$complete_vmid_full(1);
6474 }
6475
6476 sub complete_storage {
6477
6478     my $cfg = PVE::Storage::config();
6479     my $ids = $cfg->{ids};
6480
6481     my $res = [];
6482     foreach my $sid (keys %$ids) {
6483         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6484         next if !$ids->{$sid}->{content}->{images};
6485         push @$res, $sid;
6486     }
6487
6488     return $res;
6489 }
6490
6491 sub nbd_stop {
6492     my ($vmid) = @_;
6493
6494     vm_mon_cmd($vmid, 'nbd-server-stop');
6495 }
6496
6497 1;