PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::Memory;
  34 use Time::HiRes qw(gettimeofday);
  35 use File::Copy qw(copy);
  36 use URI::Escape;
  37
  38 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  39
  40 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  41
  42 # Note about locking: we use flock on the config file protect
  43 # against concurent actions.
  44 # Aditionaly, we have a 'lock' setting in the config file. This
  45 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  46 # allowed when such lock is set. But you can ignore this kind of
  47 # lock with the --skiplock flag.
  48
  49 cfs_register_file('/qemu-server/',
  50                   \&parse_vm_config,
  51                   \&write_vm_config);
  52
  53 PVE::JSONSchema::register_standard_option('skiplock', {
  54     description => "Ignore locks - only root is allowed to use this option.",
  55     type => 'boolean',
  56     optional => 1,
  57 });
  58
  59 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  60     description => "Some command save/restore state from this location.",
  61     type => 'string',
  62     maxLength => 128,
  63     optional => 1,
  64 });
  65
  66 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  67     description => "The name of the snapshot.",
  68     type => 'string', format => 'pve-configid',
  69     maxLength => 40,
  70 });
  71
  72 #no warnings 'redefine';
  73
  74 sub cgroups_write {
  75    my ($controller, $vmid, $option, $value) = @_;
  76
  77    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  78    PVE::ProcFSTools::write_proc_entry($path, $value);
  79
  80 }
  81
  82 my $nodename = PVE::INotify::nodename();
  83
  84 mkdir "/etc/pve/nodes/$nodename";
  85 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  86 mkdir $confdir;
  87
  88 my $var_run_tmpdir = "/var/run/qemu-server";
  89 mkdir $var_run_tmpdir;
  90
  91 my $lock_dir = "/var/lock/qemu-server";
  92 mkdir $lock_dir;
  93
  94 my $pcisysfs = "/sys/bus/pci";
  95
  96 my $cpu_vendor_list = {
  97     # Intel CPUs
  98     486 => 'GenuineIntel',
  99     pentium => 'GenuineIntel',
 100     pentium2  => 'GenuineIntel',
 101     pentium3  => 'GenuineIntel',
 102     coreduo => 'GenuineIntel',
 103     core2duo => 'GenuineIntel',
 104     Conroe  => 'GenuineIntel',
 105     Penryn  => 'GenuineIntel',
 106     Nehalem  => 'GenuineIntel',
 107     Westmere => 'GenuineIntel',
 108     SandyBridge => 'GenuineIntel',
 109     IvyBridge => 'GenuineIntel',
 110     Haswell => 'GenuineIntel',
 111     'Haswell-noTSX' => 'GenuineIntel',
 112     Broadwell => 'GenuineIntel',
 113     'Broadwell-noTSX' => 'GenuineIntel',
 114
 115     # AMD CPUs
 116     athlon => 'AuthenticAMD',
 117     phenom  => 'AuthenticAMD',
 118     Opteron_G1  => 'AuthenticAMD',
 119     Opteron_G2  => 'AuthenticAMD',
 120     Opteron_G3  => 'AuthenticAMD',
 121     Opteron_G4  => 'AuthenticAMD',
 122     Opteron_G5  => 'AuthenticAMD',
 123
 124     # generic types, use vendor from host node
 125     host => 'default',
 126     kvm32 => 'default',
 127     kvm64 => 'default',
 128     qemu32 => 'default',
 129     qemu64 => 'default',
 130 };
 131
 132 my $cpu_fmt = {
 133     cputype => {
 134         description => "Emulated CPU type.",
 135         type => 'string',
 136         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 137         format_description => 'cputype',
 138         default => 'kvm64',
 139         default_key => 1,
 140     },
 141     hidden => {
 142         description => "Do not identify as a KVM virtual machine.",
 143         type => 'boolean',
 144         optional => 1,
 145         default => 0
 146     },
 147 };
 148
 149 my $watchdog_fmt = {
 150     model => {
 151         default_key => 1,
 152         type => 'string',
 153         enum => [qw(i6300esb ib700)],
 154         description => "Watchdog type to emulate.",
 155         default => 'i6300esb',
 156         optional => 1,
 157     },
 158     action => {
 159         type => 'string',
 160         enum => [qw(reset shutdown poweroff pause debug none)],
 161         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 162         optional => 1,
 163     },
 164 };
 165 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 166
 167 my $confdesc = {
 168     onboot => {
 169         optional => 1,
 170         type => 'boolean',
 171         description => "Specifies whether a VM will be started during system bootup.",
 172         default => 0,
 173     },
 174     autostart => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Automatic restart after crash (currently ignored).",
 178         default => 0,
 179     },
 180     hotplug => {
 181         optional => 1,
 182         type => 'string', format => 'pve-hotplug-features',
 183         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 184         default => 'network,disk,usb',
 185     },
 186     reboot => {
 187         optional => 1,
 188         type => 'boolean',
 189         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 190         default => 1,
 191     },
 192     lock => {
 193         optional => 1,
 194         type => 'string',
 195         description => "Lock/unlock the VM.",
 196         enum => [qw(migrate backup snapshot rollback)],
 197     },
 198     cpulimit => {
 199         optional => 1,
 200         type => 'number',
 201         description => "Limit of CPU usage.",
 202         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 203         minimum => 0,
 204         maximum => 128,
 205         default => 0,
 206     },
 207     cpuunits => {
 208         optional => 1,
 209         type => 'integer',
 210         description => "CPU weight for a VM.",
 211         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 212         minimum => 0,
 213         maximum => 500000,
 214         default => 1000,
 215     },
 216     memory => {
 217         optional => 1,
 218         type => 'integer',
 219         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 220         minimum => 16,
 221         default => 512,
 222     },
 223     balloon => {
 224         optional => 1,
 225         type => 'integer',
 226         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 227         minimum => 0,
 228     },
 229     shares => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 233         minimum => 0,
 234         maximum => 50000,
 235         default => 1000,
 236     },
 237     keyboard => {
 238         optional => 1,
 239         type => 'string',
 240         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 241         enum => PVE::Tools::kvmkeymaplist(),
 242         default => 'en-us',
 243     },
 244     name => {
 245         optional => 1,
 246         type => 'string', format => 'dns-name',
 247         description => "Set a name for the VM. Only used on the configuration web interface.",
 248     },
 249     scsihw => {
 250         optional => 1,
 251         type => 'string',
 252         description => "SCSI controller model",
 253         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 254         default => 'lsi',
 255     },
 256     description => {
 257         optional => 1,
 258         type => 'string',
 259         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 260     },
 261     ostype => {
 262         optional => 1,
 263         type => 'string',
 264         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 265         description => "Specify guest operating system.",
 266         verbose_description => <<EODESC,
 267 Specify guest operating system. This is used to enable special
 268 optimization/features for specific operating systems:
 269
 270 [horizontal]
 271 other;; unspecified OS
 272 wxp;; Microsoft Windows XP
 273 w2k;; Microsoft Windows 2000
 274 w2k3;; Microsoft Windows 2003
 275 w2k8;; Microsoft Windows 2008
 276 wvista;; Microsoft Windows Vista
 277 win7;; Microsoft Windows 7
 278 win8;; Microsoft Windows 8/2012
 279 l24;; Linux 2.4 Kernel
 280 l26;; Linux 2.6/3.X Kernel
 281 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 282 EODESC
 283     },
 284     boot => {
 285         optional => 1,
 286         type => 'string',
 287         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 288         pattern => '[acdn]{1,4}',
 289         default => 'cdn',
 290     },
 291     bootdisk => {
 292         optional => 1,
 293         type => 'string', format => 'pve-qm-bootdisk',
 294         description => "Enable booting from specified disk.",
 295         pattern => '(ide|sata|scsi|virtio)\d+',
 296     },
 297     smp => {
 298         optional => 1,
 299         type => 'integer',
 300         description => "The number of CPUs. Please use option -sockets instead.",
 301         minimum => 1,
 302         default => 1,
 303     },
 304     sockets => {
 305         optional => 1,
 306         type => 'integer',
 307         description => "The number of CPU sockets.",
 308         minimum => 1,
 309         default => 1,
 310     },
 311     cores => {
 312         optional => 1,
 313         type => 'integer',
 314         description => "The number of cores per socket.",
 315         minimum => 1,
 316         default => 1,
 317     },
 318     numa => {
 319         optional => 1,
 320         type => 'boolean',
 321         description => "Enable/disable NUMA.",
 322         default => 0,
 323     },
 324     vcpus => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "Number of hotplugged vcpus.",
 328         minimum => 1,
 329         default => 0,
 330     },
 331     acpi => {
 332         optional => 1,
 333         type => 'boolean',
 334         description => "Enable/disable ACPI.",
 335         default => 1,
 336     },
 337     agent => {
 338         optional => 1,
 339         type => 'boolean',
 340         description => "Enable/disable Qemu GuestAgent.",
 341         default => 0,
 342     },
 343     kvm => {
 344         optional => 1,
 345         type => 'boolean',
 346         description => "Enable/disable KVM hardware virtualization.",
 347         default => 1,
 348     },
 349     tdf => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable time drift fix.",
 353         default => 0,
 354     },
 355     localtime => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 359     },
 360     freeze => {
 361         optional => 1,
 362         type => 'boolean',
 363         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 364     },
 365     vga => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Select the VGA type.",
 369         verbose_description => "Select the VGA type. If you want to use high resolution" .
 370             " modes (>= 1280x1024x16) then you should use the options " .
 371             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 372             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 373             "display sever. For win* OS you can select how many independent " .
 374             "displays you want, Linux guests can add displays them self. " .
 375             "You can also run without any graphic card, using a serial device" .
 376             " as terminal.",
 377         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 378     },
 379     watchdog => {
 380         optional => 1,
 381         type => 'string', format => 'pve-qm-watchdog',
 382         description => "Create a virtual hardware watchdog device.",
 383         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 384             " (by a guest action), the watchdog must be periodically polled " .
 385             "by an agent inside the guest or else the watchdog will reset " .
 386             "the guest (or execute the respective action specified)",
 387     },
 388     startdate => {
 389         optional => 1,
 390         type => 'string',
 391         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 392         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 393         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 394         default => 'now',
 395     },
 396     startup =>  get_standard_option('pve-startup-order'),
 397     template => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Enable/disable Template.",
 401         default => 0,
 402     },
 403     args => {
 404         optional => 1,
 405         type => 'string',
 406         description => "Arbitrary arguments passed to kvm.",
 407         verbose_description => <<EODESCR,
 408 Arbitrary arguments passed to kvm, for example:
 409
 410 args: -no-reboot -no-hpet
 411
 412 NOTE: this option is for experts only.
 413 EODESCR
 414     },
 415     tablet => {
 416         optional => 1,
 417         type => 'boolean',
 418         default => 1,
 419         description => "Enable/disable the USB tablet device.",
 420         verbose_description => "Enable/disable the USB tablet device. This device is " .
 421             "usually needed to allow absolute mouse positioning with VNC. " .
 422             "Else the mouse runs out of sync with normal VNC clients. " .
 423             "If you're running lots of console-only guests on one host, " .
 424             "you may consider disabling this to save some context switches. " .
 425             "This is turned off by default if you use spice (-vga=qxl).",
 426     },
 427     migrate_speed => {
 428         optional => 1,
 429         type => 'integer',
 430         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 431         minimum => 0,
 432         default => 0,
 433     },
 434     migrate_downtime => {
 435         optional => 1,
 436         type => 'number',
 437         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 438         minimum => 0,
 439         default => 0.1,
 440     },
 441     cdrom => {
 442         optional => 1,
 443         type => 'string', format => 'pve-qm-drive',
 444         typetext => 'volume',
 445         description => "This is an alias for option -ide2",
 446     },
 447     cpu => {
 448         optional => 1,
 449         description => "Emulated CPU type.",
 450         type => 'string',
 451         format => $cpu_fmt,
 452     },
 453     parent => get_standard_option('pve-snapshot-name', {
 454         optional => 1,
 455         description => "Parent snapshot name. This is used internally, and should not be modified.",
 456     }),
 457     snaptime => {
 458         optional => 1,
 459         description => "Timestamp for snapshots.",
 460         type => 'integer',
 461         minimum => 0,
 462     },
 463     vmstate => {
 464         optional => 1,
 465         type => 'string', format => 'pve-volume-id',
 466         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 467     },
 468     machine => {
 469         description => "Specific the Qemu machine type.",
 470         type => 'string',
 471         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 472         maxLength => 40,
 473         optional => 1,
 474     },
 475     smbios1 => {
 476         description => "Specify SMBIOS type 1 fields.",
 477         type => 'string', format => 'pve-qm-smbios1',
 478         maxLength => 256,
 479         optional => 1,
 480     },
 481     protection => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 485         default => 0,
 486     },
 487     bios => {
 488         optional => 1,
 489         type => 'string',
 490         enum => [ qw(seabios ovmf) ],
 491         description => "Select BIOS implementation.",
 492         default => 'seabios',
 493     },
 494 };
 495
 496 # what about other qemu settings ?
 497 #cpu => 'string',
 498 #machine => 'string',
 499 #fda => 'file',
 500 #fdb => 'file',
 501 #mtdblock => 'file',
 502 #sd => 'file',
 503 #pflash => 'file',
 504 #snapshot => 'bool',
 505 #bootp => 'file',
 506 ##tftp => 'dir',
 507 ##smb => 'dir',
 508 #kernel => 'file',
 509 #append => 'string',
 510 #initrd => 'file',
 511 ##soundhw => 'string',
 512
 513 while (my ($k, $v) = each %$confdesc) {
 514     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 515 }
 516
 517 my $MAX_IDE_DISKS = 4;
 518 my $MAX_SCSI_DISKS = 14;
 519 my $MAX_VIRTIO_DISKS = 16;
 520 my $MAX_SATA_DISKS = 6;
 521 my $MAX_USB_DEVICES = 5;
 522 my $MAX_NETS = 32;
 523 my $MAX_UNUSED_DISKS = 8;
 524 my $MAX_HOSTPCI_DEVICES = 4;
 525 my $MAX_SERIAL_PORTS = 4;
 526 my $MAX_PARALLEL_PORTS = 3;
 527 my $MAX_NUMA = 8;
 528 my $MAX_MEM = 4194304;
 529 my $STATICMEM = 1024;
 530
 531 my $numa_fmt = {
 532     cpus => {
 533         type => "string",
 534         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 535         description => "CPUs accessing this NUMA node.",
 536         format_description => "id[-id];...",
 537     },
 538     memory => {
 539         type => "number",
 540         description => "Amount of memory this NUMA node provides.",
 541         optional => 1,
 542     },
 543     hostnodes => {
 544         type => "string",
 545         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 546         description => "Host NUMA nodes to use.",
 547         format_description => "id[-id];...",
 548         optional => 1,
 549     },
 550     policy => {
 551         type => 'string',
 552         enum => [qw(preferred bind interleave)],
 553         description => "NUMA allocation policy.",
 554         optional => 1,
 555     },
 556 };
 557 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 558 my $numadesc = {
 559     optional => 1,
 560     type => 'string', format => $numa_fmt,
 561     description => "NUMA topology.",
 562 };
 563 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 564
 565 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 566     $confdesc->{"numa$i"} = $numadesc;
 567 }
 568
 569 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 570                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 571                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 572 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 573
 574 my $net_fmt_bridge_descr = <<__EOD__;
 575 Bridge to attach the network device to. The Proxmox VE standard bridge
 576 is called 'vmbr0'.
 577
 578 If you do not specify a bridge, we create a kvm user (NATed) network
 579 device, which provides DHCP and DNS services. The following addresses
 580 are used:
 581
 582  10.0.2.2   Gateway
 583  10.0.2.3   DNS Server
 584  10.0.2.4   SMB Server
 585
 586 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 587 __EOD__
 588
 589 my $net_fmt = {
 590     macaddr => {
 591         type => 'string',
 592         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 593         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 594         format_description => "XX:XX:XX:XX:XX:XX",
 595         optional => 1,
 596     },
 597     model => {
 598         type => 'string',
 599         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 600         format_description => 'model',
 601         enum => $nic_model_list,
 602         default_key => 1,
 603     },
 604     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 605     bridge => {
 606         type => 'string',
 607         description => $net_fmt_bridge_descr,
 608         format_description => 'bridge',
 609         optional => 1,
 610     },
 611     queues => {
 612         type => 'integer',
 613         minimum => 0, maximum => 16,
 614         description => 'Number of packet queues to be used on the device.',
 615         optional => 1,
 616     },
 617     rate => {
 618         type => 'number',
 619         minimum => 0,
 620         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 621         optional => 1,
 622     },
 623     tag => {
 624         type => 'integer',
 625         minimum => 1, maximum => 4094,
 626         description => 'VLAN tag to apply to packets on this interface.',
 627         optional => 1,
 628     },
 629     trunks => {
 630         type => 'string',
 631         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 632         description => 'VLAN trunks to pass through this interface.',
 633         format_description => 'vlanid[;vlanid...]',
 634         optional => 1,
 635     },
 636     firewall => {
 637         type => 'boolean',
 638         description => 'Whether this interface should be protected by the firewall.',
 639         optional => 1,
 640     },
 641     link_down => {
 642         type => 'boolean',
 643         description => 'Whether this interface should be disconnected (like pulling the plug).',
 644         optional => 1,
 645     },
 646 };
 647
 648 my $netdesc = {
 649     optional => 1,
 650     type => 'string', format => $net_fmt,
 651     description => "Specify network devices.",
 652 };
 653
 654 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 655
 656 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 657     $confdesc->{"net$i"} = $netdesc;
 658 }
 659
 660 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 661 sub verify_volume_id_or_qm_path {
 662     my ($volid, $noerr) = @_;
 663
 664     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 665         return $volid;
 666     }
 667
 668     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 669     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 670     if ($@) {
 671         return undef if $noerr;
 672         die $@;
 673     }
 674     return $volid;
 675 }
 676
 677 my $drivename_hash;
 678
 679 my %drivedesc_base = (
 680     volume => { alias => 'file' },
 681     file => {
 682         type => 'string',
 683         format => 'pve-volume-id-or-qm-path',
 684         default_key => 1,
 685         format_description => 'volume',
 686         description => "The drive's backing volume.",
 687     },
 688     media => {
 689         type => 'string',
 690         enum => [qw(cdrom disk)],
 691         description => "The drive's media type.",
 692         default => 'disk',
 693         optional => 1
 694     },
 695     cyls => {
 696         type => 'integer',
 697         description => "Force the drive's physical geometry to have a specific cylinder count.",
 698         optional => 1
 699     },
 700     heads => {
 701         type => 'integer',
 702         description => "Force the drive's physical geometry to have a specific head count.",
 703         optional => 1
 704     },
 705     secs => {
 706         type => 'integer',
 707         description => "Force the drive's physical geometry to have a specific sector count.",
 708         optional => 1
 709     },
 710     trans => {
 711         type => 'string',
 712         enum => [qw(none lba auto)],
 713         description => "Force disk geometry bios translation mode.",
 714         optional => 1,
 715     },
 716     snapshot => {
 717         type => 'boolean',
 718         description => "Whether the drive should be included when making snapshots.",
 719         optional => 1,
 720     },
 721     cache => {
 722         type => 'string',
 723         enum => [qw(none writethrough writeback unsafe directsync)],
 724         description => "The drive's cache mode",
 725         optional => 1,
 726     },
 727     format => {
 728         type => 'string',
 729         format_description => 'image format',
 730         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 731         description => "The drive's backing file's data format.",
 732         optional => 1,
 733     },
 734     size => {
 735         type => 'string',
 736         format => 'disk-size',
 737         format_description => 'DiskSize',
 738         description => "Disk size. This is purely informational and has no effect.",
 739         optional => 1,
 740     },
 741     backup => {
 742         type => 'boolean',
 743         description => "Whether the drive should be included when making backups.",
 744         optional => 1,
 745     },
 746     werror => {
 747         type => 'string',
 748         enum => [qw(enospc ignore report stop)],
 749         description => 'Write error action.',
 750         optional => 1,
 751     },
 752     aio => {
 753         type => 'string',
 754         enum => [qw(native threads)],
 755         description => 'AIO type to use.',
 756         optional => 1,
 757     },
 758     discard => {
 759         type => 'string',
 760         enum => [qw(ignore on)],
 761         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 762         optional => 1,
 763     },
 764     detect_zeroes => {
 765         type => 'boolean',
 766         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 767         optional => 1,
 768     },
 769     serial => {
 770         type => 'string',
 771         format => 'urlencoded',
 772         format_description => 'serial',
 773         maxLength => 20*3, # *3 since it's %xx url enoded
 774         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 775         optional => 1,
 776     }
 777 );
 778
 779 my %rerror_fmt = (
 780     rerror => {
 781         type => 'string',
 782         enum => [qw(ignore report stop)],
 783         description => 'Read error action.',
 784         optional => 1,
 785     },
 786 );
 787
 788 my %iothread_fmt = ( iothread => {
 789         type => 'boolean',
 790         description => "Whether to use iothreads for this drive",
 791         optional => 1,
 792 });
 793
 794 my %model_fmt = (
 795     model => {
 796         type => 'string',
 797         format => 'urlencoded',
 798         format_description => 'model',
 799         maxLength => 40*3, # *3 since it's %xx url enoded
 800         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 801         optional => 1,
 802     },
 803 );
 804
 805 my %queues_fmt = (
 806     queues => {
 807         type => 'integer',
 808         description => "Number of queues.",
 809         minimum => 2,
 810         optional => 1
 811     }
 812 );
 813
 814 my $add_throttle_desc = sub {
 815     my ($key, $type, $what, $unit, $longunit) = @_;
 816     $drivedesc_base{$key} = {
 817         type => $type,
 818         format_description => $unit,
 819         description => "Maximum $what speed in $longunit per second.",
 820         optional => 1,
 821     };
 822 };
 823 # throughput: (leaky bucket)
 824 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 825 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 826 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 827 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 828 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 829 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 830 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 831 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 832 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 833
 834 # pools: (pool of IO before throttling starts taking effect)
 835 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 836 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 837 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 838 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 839 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 840 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 841
 842 my $ide_fmt = {
 843     %drivedesc_base,
 844     %rerror_fmt,
 845     %model_fmt,
 846 };
 847
 848 my $idedesc = {
 849     optional => 1,
 850     type => 'string', format => $ide_fmt,
 851     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 852 };
 853 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 854
 855 my $scsi_fmt = {
 856     %drivedesc_base,
 857     %iothread_fmt,
 858     %queues_fmt,
 859 };
 860 my $scsidesc = {
 861     optional => 1,
 862     type => 'string', format => $scsi_fmt,
 863     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 864 };
 865 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 866
 867 my $sata_fmt = {
 868     %drivedesc_base,
 869     %rerror_fmt,
 870 };
 871 my $satadesc = {
 872     optional => 1,
 873     type => 'string', format => $sata_fmt,
 874     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 875 };
 876 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 877
 878 my $virtio_fmt = {
 879     %drivedesc_base,
 880     %iothread_fmt,
 881     %rerror_fmt,
 882 };
 883 my $virtiodesc = {
 884     optional => 1,
 885     type => 'string', format => $virtio_fmt,
 886     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 887 };
 888 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 889
 890 my $alldrive_fmt = {
 891     %drivedesc_base,
 892     %rerror_fmt,
 893     %iothread_fmt,
 894     %model_fmt,
 895     %queues_fmt,
 896 };
 897
 898 my $usb_fmt = {
 899     host => {
 900         default_key => 1,
 901         type => 'string', format => 'pve-qm-usb-device',
 902         format_description => 'HOSTUSBDEVICE|spice',
 903         description => <<EODESCR,
 904 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 905
 906  'bus-port(.port)*' (decimal numbers) or
 907  'vendor_id:product_id' (hexadeciaml numbers) or
 908  'spice'
 909
 910 You can use the 'lsusb -t' command to list existing usb devices.
 911
 912 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 913
 914 The value 'spice' can be used to add a usb redirection devices for spice.
 915 EODESCR
 916     },
 917     usb3 => {
 918         optional => 1,
 919         type => 'boolean',
 920         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 921         default => 0,
 922     },
 923 };
 924
 925 my $usbdesc = {
 926     optional => 1,
 927     type => 'string', format => $usb_fmt,
 928     description => "Configure an USB device (n is 0 to 4).",
 929 };
 930 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 931
 932 # NOTE: the match-groups of this regex are used in parse_hostpci
 933 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 934 my $hostpci_fmt = {
 935     host => {
 936         default_key => 1,
 937         type => 'string',
 938         pattern => qr/$PCIRE(;$PCIRE)*/,
 939         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 940         description => <<EODESCR,
 941 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 942 of PCI virtual functions of the host. HOSTPCIID syntax is:
 943
 944 'bus:dev.func' (hexadecimal numbers)
 945
 946 You can us the 'lspci' command to list existing PCI devices.
 947 EODESCR
 948     },
 949     rombar => {
 950         type => 'boolean',
 951         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 952         optional => 1,
 953         default => 1,
 954     },
 955     pcie => {
 956         type => 'boolean',
 957         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
 958         optional => 1,
 959         default => 0,
 960     },
 961     'x-vga' => {
 962         type => 'boolean',
 963         description =>  "Enable vfio-vga device support.",
 964         optional => 1,
 965         default => 0,
 966     },
 967 };
 968 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
 969
 970 my $hostpcidesc = {
 971         optional => 1,
 972         type => 'string', format => 'pve-qm-hostpci',
 973         description => "Map host PCI devices into guest.",
 974         verbose_description =>  <<EODESCR,
 975 Map host PCI devices into guest.
 976
 977 NOTE: This option allows direct access to host hardware. So it is no longer
 978 possible to migrate such machines - use with special care.
 979
 980 CAUTION: Experimental! User reported problems with this option.
 981 EODESCR
 982 };
 983 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 984
 985 my $serialdesc = {
 986         optional => 1,
 987         type => 'string',
 988         pattern => '(/dev/.+|socket)',
 989         description =>  "Create a serial device inside the VM (n is 0 to 3)",
 990         verbose_description =>  <<EODESCR,
 991 Create a serial device inside the VM (n is 0 to 3), and pass through a
 992 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
 993 host side (use 'qm terminal' to open a terminal connection).
 994
 995 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 996
 997 CAUTION: Experimental! User reported problems with this option.
 998 EODESCR
 999 };
1000
1001 my $paralleldesc= {
1002         optional => 1,
1003         type => 'string',
1004         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1005         description =>  "Map host parallel devices (n is 0 to 2).",
1006         verbose_description =>  <<EODESCR,
1007 Map host parallel devices (n is 0 to 2).
1008
1009 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1010
1011 CAUTION: Experimental! User reported problems with this option.
1012 EODESCR
1013 };
1014
1015 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1016     $confdesc->{"parallel$i"} = $paralleldesc;
1017 }
1018
1019 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1020     $confdesc->{"serial$i"} = $serialdesc;
1021 }
1022
1023 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1024     $confdesc->{"hostpci$i"} = $hostpcidesc;
1025 }
1026
1027 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1028     $drivename_hash->{"ide$i"} = 1;
1029     $confdesc->{"ide$i"} = $idedesc;
1030 }
1031
1032 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1033     $drivename_hash->{"sata$i"} = 1;
1034     $confdesc->{"sata$i"} = $satadesc;
1035 }
1036
1037 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1038     $drivename_hash->{"scsi$i"} = 1;
1039     $confdesc->{"scsi$i"} = $scsidesc ;
1040 }
1041
1042 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1043     $drivename_hash->{"virtio$i"} = 1;
1044     $confdesc->{"virtio$i"} = $virtiodesc;
1045 }
1046
1047 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1048     $confdesc->{"usb$i"} = $usbdesc;
1049 }
1050
1051 my $unuseddesc = {
1052     optional => 1,
1053     type => 'string', format => 'pve-volume-id',
1054     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1055 };
1056
1057 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1058     $confdesc->{"unused$i"} = $unuseddesc;
1059 }
1060
1061 my $kvm_api_version = 0;
1062
1063 sub kvm_version {
1064
1065     return $kvm_api_version if $kvm_api_version;
1066
1067     my $fh = IO::File->new("</dev/kvm") ||
1068         return 0;
1069
1070     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1071         $kvm_api_version = $v;
1072     }
1073
1074     $fh->close();
1075
1076     return  $kvm_api_version;
1077 }
1078
1079 my $kvm_user_version;
1080
1081 sub kvm_user_version {
1082
1083     return $kvm_user_version if $kvm_user_version;
1084
1085     $kvm_user_version = 'unknown';
1086
1087     my $code = sub {
1088         my $line = shift;
1089         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1090             $kvm_user_version = $2;
1091         }
1092     };
1093
1094     eval { run_command("kvm -version", outfunc => $code); };
1095     warn $@ if $@;
1096
1097     return $kvm_user_version;
1098
1099 }
1100
1101 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1102
1103 sub valid_drive_names {
1104     # order is important - used to autoselect boot disk
1105     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1106             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1107             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1108             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1109 }
1110
1111 sub is_valid_drivename {
1112     my $dev = shift;
1113
1114     return defined($drivename_hash->{$dev});
1115 }
1116
1117 sub option_exists {
1118     my $key = shift;
1119     return defined($confdesc->{$key});
1120 }
1121
1122 sub nic_models {
1123     return $nic_model_list;
1124 }
1125
1126 sub os_list_description {
1127
1128     return {
1129         other => 'Other',
1130         wxp => 'Windows XP',
1131         w2k => 'Windows 2000',
1132         w2k3 =>, 'Windows 2003',
1133         w2k8 => 'Windows 2008',
1134         wvista => 'Windows Vista',
1135         win7 => 'Windows 7',
1136         win8 => 'Windows 8/2012',
1137         l24 => 'Linux 2.4',
1138         l26 => 'Linux 2.6',
1139     };
1140 }
1141
1142 my $cdrom_path;
1143
1144 sub get_cdrom_path {
1145
1146     return  $cdrom_path if $cdrom_path;
1147
1148     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1149     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1150     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1151 }
1152
1153 sub get_iso_path {
1154     my ($storecfg, $vmid, $cdrom) = @_;
1155
1156     if ($cdrom eq 'cdrom') {
1157         return get_cdrom_path();
1158     } elsif ($cdrom eq 'none') {
1159         return '';
1160     } elsif ($cdrom =~ m|^/|) {
1161         return $cdrom;
1162     } else {
1163         return PVE::Storage::path($storecfg, $cdrom);
1164     }
1165 }
1166
1167 # try to convert old style file names to volume IDs
1168 sub filename_to_volume_id {
1169     my ($vmid, $file, $media) = @_;
1170
1171     if (!($file eq 'none' || $file eq 'cdrom' ||
1172           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1173
1174         return undef if $file =~ m|/|;
1175
1176         if ($media && $media eq 'cdrom') {
1177             $file = "local:iso/$file";
1178         } else {
1179             $file = "local:$vmid/$file";
1180         }
1181     }
1182
1183     return $file;
1184 }
1185
1186 sub verify_media_type {
1187     my ($opt, $vtype, $media) = @_;
1188
1189     return if !$media;
1190
1191     my $etype;
1192     if ($media eq 'disk') {
1193         $etype = 'images';
1194     } elsif ($media eq 'cdrom') {
1195         $etype = 'iso';
1196     } else {
1197         die "internal error";
1198     }
1199
1200     return if ($vtype eq $etype);
1201
1202     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1203 }
1204
1205 sub cleanup_drive_path {
1206     my ($opt, $storecfg, $drive) = @_;
1207
1208     # try to convert filesystem paths to volume IDs
1209
1210     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1211         ($drive->{file} !~ m|^/dev/.+|) &&
1212         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1213         ($drive->{file} !~ m/^\d+$/)) {
1214         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1215         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1216         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1217         verify_media_type($opt, $vtype, $drive->{media});
1218         $drive->{file} = $volid;
1219     }
1220
1221     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1222 }
1223
1224 sub parse_hotplug_features {
1225     my ($data) = @_;
1226
1227     my $res = {};
1228
1229     return $res if $data eq '0';
1230
1231     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1232
1233     foreach my $feature (PVE::Tools::split_list($data)) {
1234         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1235             $res->{$1} = 1;
1236         } else {
1237             warn "ignoring unknown hotplug feature '$feature'\n";
1238         }
1239     }
1240     return $res;
1241 }
1242
1243 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1244 sub pve_verify_hotplug_features {
1245     my ($value, $noerr) = @_;
1246
1247     return $value if parse_hotplug_features($value);
1248
1249     return undef if $noerr;
1250
1251     die "unable to parse hotplug option\n";
1252 }
1253
1254 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1255 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1256 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1257 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1258 #        [,iothread=on][,serial=serial][,model=model]
1259
1260 sub parse_drive {
1261     my ($key, $data) = @_;
1262
1263     my ($interface, $index);
1264
1265     if ($key =~ m/^([^\d]+)(\d+)$/) {
1266         $interface = $1;
1267         $index = $2;
1268     } else {
1269         return undef;
1270     }
1271
1272     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1273                                      : $confdesc->{$key}->{format};
1274     if (!$desc) {
1275         warn "invalid drive key: $key\n";
1276         return undef;
1277     }
1278     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1279     return undef if !$res;
1280     $res->{interface} = $interface;
1281     $res->{index} = $index;
1282
1283     my $error = 0;
1284     foreach my $opt (qw(bps bps_rd bps_wr)) {
1285         if (my $bps = defined(delete $res->{$opt})) {
1286             if (defined($res->{"m$opt"})) {
1287                 warn "both $opt and m$opt specified\n";
1288                 ++$error;
1289                 next;
1290             }
1291             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1292         }
1293     }
1294     return undef if $error;
1295
1296     return undef if $res->{mbps_rd} && $res->{mbps};
1297     return undef if $res->{mbps_wr} && $res->{mbps};
1298     return undef if $res->{iops_rd} && $res->{iops};
1299     return undef if $res->{iops_wr} && $res->{iops};
1300
1301     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1302         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1303         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1304         return undef if $res->{interface} eq 'virtio';
1305     }
1306
1307     if (my $size = $res->{size}) {
1308         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1309     }
1310
1311     return $res;
1312 }
1313
1314 sub print_drive {
1315     my ($vmid, $drive) = @_;
1316     my $data = { %$drive };
1317     delete $data->{$_} for qw(index interface);
1318     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1319 }
1320
1321 sub scsi_inquiry {
1322     my($fh, $noerr) = @_;
1323
1324     my $SG_IO = 0x2285;
1325     my $SG_GET_VERSION_NUM = 0x2282;
1326
1327     my $versionbuf = "\x00" x 8;
1328     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1329     if (!$ret) {
1330         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1331         return undef;
1332     }
1333     my $version = unpack("I", $versionbuf);
1334     if ($version < 30000) {
1335         die "scsi generic interface too old\n"  if !$noerr;
1336         return undef;
1337     }
1338
1339     my $buf = "\x00" x 36;
1340     my $sensebuf = "\x00" x 8;
1341     my $cmd = pack("C x3 C x1", 0x12, 36);
1342
1343     # see /usr/include/scsi/sg.h
1344     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1345
1346     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1347                       length($sensebuf), 0, length($buf), $buf,
1348                       $cmd, $sensebuf, 6000);
1349
1350     $ret = ioctl($fh, $SG_IO, $packet);
1351     if (!$ret) {
1352         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1353         return undef;
1354     }
1355
1356     my @res = unpack($sg_io_hdr_t, $packet);
1357     if ($res[17] || $res[18]) {
1358         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1359         return undef;
1360     }
1361
1362     my $res = {};
1363     (my $byte0, my $byte1, $res->{vendor},
1364      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1365
1366     $res->{removable} = $byte1 & 128 ? 1 : 0;
1367     $res->{type} = $byte0 & 31;
1368
1369     return $res;
1370 }
1371
1372 sub path_is_scsi {
1373     my ($path) = @_;
1374
1375     my $fh = IO::File->new("+<$path") || return undef;
1376     my $res = scsi_inquiry($fh, 1);
1377     close($fh);
1378
1379     return $res;
1380 }
1381
1382 sub machine_type_is_q35 {
1383     my ($conf) = @_;
1384
1385     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1386 }
1387
1388 sub print_tabletdevice_full {
1389     my ($conf) = @_;
1390
1391     my $q35 = machine_type_is_q35($conf);
1392
1393     # we use uhci for old VMs because tablet driver was buggy in older qemu
1394     my $usbbus = $q35 ? "ehci" : "uhci";
1395
1396     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1397 }
1398
1399 sub print_drivedevice_full {
1400     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1401
1402     my $device = '';
1403     my $maxdev = 0;
1404
1405     if ($drive->{interface} eq 'virtio') {
1406         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1407         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1408         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1409     } elsif ($drive->{interface} eq 'scsi') {
1410
1411         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1412         my $unit = $drive->{index} % $maxdev;
1413         my $devicetype = 'hd';
1414         my $path = '';
1415         if (drive_is_cdrom($drive)) {
1416             $devicetype = 'cd';
1417         } else {
1418             if ($drive->{file} =~ m|^/|) {
1419                 $path = $drive->{file};
1420                 if (my $info = path_is_scsi($path)) {
1421                     if ($info->{type} == 0) {
1422                         $devicetype = 'block';
1423                     } elsif ($info->{type} == 1) { # tape
1424                         $devicetype = 'generic';
1425                     }
1426                 }
1427             } else {
1428                  $path = PVE::Storage::path($storecfg, $drive->{file});
1429             }
1430
1431             if($path =~ m/^iscsi\:\/\//){
1432                 $devicetype = 'generic';
1433             }
1434         }
1435
1436         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1437             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1438         } else {
1439             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1440         }
1441
1442     } elsif ($drive->{interface} eq 'ide'){
1443         $maxdev = 2;
1444         my $controller = int($drive->{index} / $maxdev);
1445         my $unit = $drive->{index} % $maxdev;
1446         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1447
1448         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1449         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1450             $model = URI::Escape::uri_unescape($model);
1451             $device .= ",model=$model";
1452         }
1453     } elsif ($drive->{interface} eq 'sata'){
1454         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1455         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1456         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1457     } elsif ($drive->{interface} eq 'usb') {
1458         die "implement me";
1459         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1460     } else {
1461         die "unsupported interface type";
1462     }
1463
1464     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1465
1466     return $device;
1467 }
1468
1469 sub get_initiator_name {
1470     my $initiator;
1471
1472     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1473     while (defined(my $line = <$fh>)) {
1474         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1475         $initiator = $1;
1476         last;
1477     }
1478     $fh->close();
1479
1480     return $initiator;
1481 }
1482
1483 sub print_drive_full {
1484     my ($storecfg, $vmid, $drive) = @_;
1485
1486     my $path;
1487     my $volid = $drive->{file};
1488     my $format;
1489
1490     if (drive_is_cdrom($drive)) {
1491         $path = get_iso_path($storecfg, $vmid, $volid);
1492     } else {
1493         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1494         if ($storeid) {
1495             $path = PVE::Storage::path($storecfg, $volid);
1496             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1497             $format = qemu_img_format($scfg, $volname);
1498         } else {
1499             $path = $volid;
1500             $format = "raw";
1501         }
1502    }
1503
1504     my $opts = '';
1505     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1506     foreach my $o (@qemu_drive_options) {
1507         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1508     }
1509     if (my $serial = $drive->{serial}) {
1510         $serial = URI::Escape::uri_unescape($serial);
1511         $opts .= ",serial=$serial";
1512     }
1513
1514     $opts .= ",format=$format" if $format && !$drive->{format};
1515
1516     foreach my $o (qw(bps bps_rd bps_wr)) {
1517         my $v = $drive->{"m$o"};
1518         $opts .= ",$o=" . int($v*1024*1024) if $v;
1519     }
1520
1521     my $cache_direct = 0;
1522
1523     if (my $cache = $drive->{cache}) {
1524         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1525     } elsif (!drive_is_cdrom($drive)) {
1526         $opts .= ",cache=none";
1527         $cache_direct = 1;
1528     }
1529
1530     # aio native works only with O_DIRECT
1531     if (!$drive->{aio}) {
1532         if($cache_direct) {
1533             $opts .= ",aio=native";
1534         } else {
1535             $opts .= ",aio=threads";
1536         }
1537     }
1538
1539     if (!drive_is_cdrom($drive)) {
1540         my $detectzeroes;
1541         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1542             $detectzeroes = 'off';
1543         } elsif ($drive->{discard}) {
1544             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1545         } else {
1546             # This used to be our default with discard not being specified:
1547             $detectzeroes = 'on';
1548         }
1549         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1550     }
1551
1552     my $pathinfo = $path ? "file=$path," : '';
1553
1554     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1555 }
1556
1557 sub print_netdevice_full {
1558     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1559
1560     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1561
1562     my $device = $net->{model};
1563     if ($net->{model} eq 'virtio') {
1564          $device = 'virtio-net-pci';
1565      };
1566
1567     my $pciaddr = print_pci_addr("$netid", $bridges);
1568     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1569     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1570         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1571         my $vectors = $net->{queues} * 2 + 2;
1572         $tmpstr .= ",vectors=$vectors,mq=on";
1573     }
1574     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1575
1576     if ($use_old_bios_files) {
1577         my $romfile;
1578         if ($device eq 'virtio-net-pci') {
1579             $romfile = 'pxe-virtio.rom';
1580         } elsif ($device eq 'e1000') {
1581             $romfile = 'pxe-e1000.rom';
1582         } elsif ($device eq 'ne2k') {
1583             $romfile = 'pxe-ne2k_pci.rom';
1584         } elsif ($device eq 'pcnet') {
1585             $romfile = 'pxe-pcnet.rom';
1586         } elsif ($device eq 'rtl8139') {
1587             $romfile = 'pxe-rtl8139.rom';
1588         }
1589         $tmpstr .= ",romfile=$romfile" if $romfile;
1590     }
1591
1592     return $tmpstr;
1593 }
1594
1595 sub print_netdev_full {
1596     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1597
1598     my $i = '';
1599     if ($netid =~ m/^net(\d+)$/) {
1600         $i = int($1);
1601     }
1602
1603     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1604
1605     my $ifname = "tap${vmid}i$i";
1606
1607     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1608     die "interface name '$ifname' is too long (max 15 character)\n"
1609         if length($ifname) >= 16;
1610
1611     my $vhostparam = '';
1612     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1613
1614     my $vmname = $conf->{name} || "vm$vmid";
1615
1616     my $netdev = "";
1617     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1618
1619     if ($net->{bridge}) {
1620         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1621     } else {
1622         $netdev = "type=user,id=$netid,hostname=$vmname";
1623     }
1624
1625     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1626
1627     return $netdev;
1628 }
1629
1630 sub drive_is_cdrom {
1631     my ($drive) = @_;
1632
1633     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1634
1635 }
1636
1637 sub parse_number_sets {
1638     my ($set) = @_;
1639     my $res = [];
1640     foreach my $part (split(/;/, $set)) {
1641         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1642             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1643             push @$res, [ $1, $2 ];
1644         } else {
1645             die "invalid range: $part\n";
1646         }
1647     }
1648     return $res;
1649 }
1650
1651 sub parse_numa {
1652     my ($data) = @_;
1653
1654     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1655     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1656     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1657     return $res;
1658 }
1659
1660 sub parse_hostpci {
1661     my ($value) = @_;
1662
1663     return undef if !$value;
1664
1665     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1666
1667     my @idlist = split(/;/, $res->{host});
1668     delete $res->{host};
1669     foreach my $id (@idlist) {
1670         if ($id =~ /^$PCIRE$/) {
1671             push @{$res->{pciid}}, { id => $1, function => ($2//'0') };
1672         } else {
1673             # should have been caught by parse_property_string already
1674             die "failed to parse PCI id: $id\n";
1675         }
1676     }
1677     return $res;
1678 }
1679
1680 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1681 sub parse_net {
1682     my ($data) = @_;
1683
1684     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1685     if ($@) {
1686         warn $@;
1687         return undef;
1688     }
1689     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1690     return $res;
1691 }
1692
1693 sub print_net {
1694     my $net = shift;
1695
1696     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1697 }
1698
1699 sub add_random_macs {
1700     my ($settings) = @_;
1701
1702     foreach my $opt (keys %$settings) {
1703         next if $opt !~ m/^net(\d+)$/;
1704         my $net = parse_net($settings->{$opt});
1705         next if !$net;
1706         $settings->{$opt} = print_net($net);
1707     }
1708 }
1709
1710 sub vm_is_volid_owner {
1711     my ($storecfg, $vmid, $volid) = @_;
1712
1713     if ($volid !~  m|^/|) {
1714         my ($path, $owner);
1715         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1716         if ($owner && ($owner == $vmid)) {
1717             return 1;
1718         }
1719     }
1720
1721     return undef;
1722 }
1723
1724 sub split_flagged_list {
1725     my $text = shift || '';
1726     $text =~ s/[,;]/ /g;
1727     $text =~ s/^\s+//;
1728     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1729 }
1730
1731 sub join_flagged_list {
1732     my ($how, $lst) = @_;
1733     join $how, map { $lst->{$_} . $_ } keys %$lst;
1734 }
1735
1736 sub vmconfig_delete_pending_option {
1737     my ($conf, $key, $force) = @_;
1738
1739     delete $conf->{pending}->{$key};
1740     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1741     $pending_delete_hash->{$key} = $force ? '!' : '';
1742     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1743 }
1744
1745 sub vmconfig_undelete_pending_option {
1746     my ($conf, $key) = @_;
1747
1748     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1749     delete $pending_delete_hash->{$key};
1750
1751     if (%$pending_delete_hash) {
1752         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1753     } else {
1754         delete $conf->{pending}->{delete};
1755     }
1756 }
1757
1758 sub vmconfig_register_unused_drive {
1759     my ($storecfg, $vmid, $conf, $drive) = @_;
1760
1761     if (!drive_is_cdrom($drive)) {
1762         my $volid = $drive->{file};
1763         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1764             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1765         }
1766     }
1767 }
1768
1769 sub vmconfig_cleanup_pending {
1770     my ($conf) = @_;
1771
1772     # remove pending changes when nothing changed
1773     my $changes;
1774     foreach my $opt (keys %{$conf->{pending}}) {
1775         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1776             $changes = 1;
1777             delete $conf->{pending}->{$opt};
1778         }
1779     }
1780
1781     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1782     my $pending_delete_hash = {};
1783     while (my ($opt, $force) = each %$current_delete_hash) {
1784         if (defined($conf->{$opt})) {
1785             $pending_delete_hash->{$opt} = $force;
1786         } else {
1787             $changes = 1;
1788         }
1789     }
1790
1791     if (%$pending_delete_hash) {
1792         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1793     } else {
1794         delete $conf->{pending}->{delete};
1795     }
1796
1797     return $changes;
1798 }
1799
1800 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1801 my $smbios1_fmt = {
1802     uuid => {
1803         type => 'string',
1804         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1805         format_description => 'UUID',
1806         description => "Set SMBIOS1 UUID.",
1807         optional => 1,
1808     },
1809     version => {
1810         type => 'string',
1811         pattern => '\S+',
1812         format_description => 'string',
1813         description => "Set SMBIOS1 version.",
1814         optional => 1,
1815     },
1816     serial => {
1817         type => 'string',
1818         pattern => '\S+',
1819         format_description => 'string',
1820         description => "Set SMBIOS1 serial number.",
1821         optional => 1,
1822     },
1823     manufacturer => {
1824         type => 'string',
1825         pattern => '\S+',
1826         format_description => 'string',
1827         description => "Set SMBIOS1 manufacturer.",
1828         optional => 1,
1829     },
1830     product => {
1831         type => 'string',
1832         pattern => '\S+',
1833         format_description => 'string',
1834         description => "Set SMBIOS1 product ID.",
1835         optional => 1,
1836     },
1837     sku => {
1838         type => 'string',
1839         pattern => '\S+',
1840         format_description => 'string',
1841         description => "Set SMBIOS1 SKU string.",
1842         optional => 1,
1843     },
1844     family => {
1845         type => 'string',
1846         pattern => '\S+',
1847         format_description => 'string',
1848         description => "Set SMBIOS1 family string.",
1849         optional => 1,
1850     },
1851 };
1852
1853 sub parse_smbios1 {
1854     my ($data) = @_;
1855
1856     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1857     warn $@ if $@;
1858     return $res;
1859 }
1860
1861 sub print_smbios1 {
1862     my ($smbios1) = @_;
1863     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1864 }
1865
1866 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1867
1868 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1869 sub verify_bootdisk {
1870     my ($value, $noerr) = @_;
1871
1872     return $value if is_valid_drivename($value);
1873
1874     return undef if $noerr;
1875
1876     die "invalid boot disk '$value'\n";
1877 }
1878
1879 sub parse_watchdog {
1880     my ($value) = @_;
1881
1882     return undef if !$value;
1883
1884     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1885     warn $@ if $@;
1886     return $res;
1887 }
1888
1889 sub parse_usb_device {
1890     my ($value) = @_;
1891
1892     return undef if !$value;
1893
1894     my $res = {};
1895     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1896         $res->{vendorid} = $2;
1897         $res->{productid} = $4;
1898     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1899         $res->{hostbus} = $1;
1900         $res->{hostport} = $2;
1901     } elsif ($value =~ m/^spice$/i) {
1902         $res->{spice} = 1;
1903     } else {
1904         return undef;
1905     }
1906
1907     return $res;
1908 }
1909
1910 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1911 sub verify_usb_device {
1912     my ($value, $noerr) = @_;
1913
1914     return $value if parse_usb_device($value);
1915
1916     return undef if $noerr;
1917
1918     die "unable to parse usb device\n";
1919 }
1920
1921 # add JSON properties for create and set function
1922 sub json_config_properties {
1923     my $prop = shift;
1924
1925     foreach my $opt (keys %$confdesc) {
1926         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1927         $prop->{$opt} = $confdesc->{$opt};
1928     }
1929
1930     return $prop;
1931 }
1932
1933 sub check_type {
1934     my ($key, $value) = @_;
1935
1936     die "unknown setting '$key'\n" if !$confdesc->{$key};
1937
1938     my $type = $confdesc->{$key}->{type};
1939
1940     if (!defined($value)) {
1941         die "got undefined value\n";
1942     }
1943
1944     if ($value =~ m/[\n\r]/) {
1945         die "property contains a line feed\n";
1946     }
1947
1948     if ($type eq 'boolean') {
1949         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1950         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1951         die "type check ('boolean') failed - got '$value'\n";
1952     } elsif ($type eq 'integer') {
1953         return int($1) if $value =~ m/^(\d+)$/;
1954         die "type check ('integer') failed - got '$value'\n";
1955     } elsif ($type eq 'number') {
1956         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1957         die "type check ('number') failed - got '$value'\n";
1958     } elsif ($type eq 'string') {
1959         if (my $fmt = $confdesc->{$key}->{format}) {
1960             if ($fmt eq 'pve-qm-drive') {
1961                 # special case - we need to pass $key to parse_drive()
1962                 my $drive = parse_drive($key, $value);
1963                 return $value if $drive;
1964                 die "unable to parse drive options\n";
1965             }
1966             PVE::JSONSchema::check_format($fmt, $value);
1967             return $value;
1968         }
1969         $value =~ s/^\"(.*)\"$/$1/;
1970         return $value;
1971     } else {
1972         die "internal error"
1973     }
1974 }
1975
1976 sub check_iommu_support{
1977     #fixme : need to check IOMMU support
1978     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1979
1980     my $iommu=1;
1981     return $iommu;
1982
1983 }
1984
1985 sub touch_config {
1986     my ($vmid) = @_;
1987
1988     my $conf = PVE::QemuConfig->config_file($vmid);
1989     utime undef, undef, $conf;
1990 }
1991
1992 sub destroy_vm {
1993     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1994
1995     my $conffile = PVE::QemuConfig->config_file($vmid);
1996
1997     my $conf = PVE::QemuConfig->load_config($vmid);
1998
1999     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2000
2001     # only remove disks owned by this VM
2002     foreach_drive($conf, sub {
2003         my ($ds, $drive) = @_;
2004
2005         return if drive_is_cdrom($drive);
2006
2007         my $volid = $drive->{file};
2008
2009         return if !$volid || $volid =~ m|^/|;
2010
2011         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2012         return if !$path || !$owner || ($owner != $vmid);
2013
2014         PVE::Storage::vdisk_free($storecfg, $volid);
2015     });
2016
2017     if ($keep_empty_config) {
2018         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2019     } else {
2020         unlink $conffile;
2021     }
2022
2023     # also remove unused disk
2024     eval {
2025         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2026
2027         eval {
2028             PVE::Storage::foreach_volid($dl, sub {
2029                 my ($volid, $sid, $volname, $d) = @_;
2030                 PVE::Storage::vdisk_free($storecfg, $volid);
2031             });
2032         };
2033         warn $@ if $@;
2034
2035     };
2036     warn $@ if $@;
2037 }
2038
2039 sub parse_vm_config {
2040     my ($filename, $raw) = @_;
2041
2042     return undef if !defined($raw);
2043
2044     my $res = {
2045         digest => Digest::SHA::sha1_hex($raw),
2046         snapshots => {},
2047         pending => {},
2048     };
2049
2050     $filename =~ m|/qemu-server/(\d+)\.conf$|
2051         || die "got strange filename '$filename'";
2052
2053     my $vmid = $1;
2054
2055     my $conf = $res;
2056     my $descr;
2057     my $section = '';
2058
2059     my @lines = split(/\n/, $raw);
2060     foreach my $line (@lines) {
2061         next if $line =~ m/^\s*$/;
2062
2063         if ($line =~ m/^\[PENDING\]\s*$/i) {
2064             $section = 'pending';
2065             if (defined($descr)) {
2066                 $descr =~ s/\s+$//;
2067                 $conf->{description} = $descr;
2068             }
2069             $descr = undef;
2070             $conf = $res->{$section} = {};
2071             next;
2072
2073         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2074             $section = $1;
2075             if (defined($descr)) {
2076                 $descr =~ s/\s+$//;
2077                 $conf->{description} = $descr;
2078             }
2079             $descr = undef;
2080             $conf = $res->{snapshots}->{$section} = {};
2081             next;
2082         }
2083
2084         if ($line =~ m/^\#(.*)\s*$/) {
2085             $descr = '' if !defined($descr);
2086             $descr .= PVE::Tools::decode_text($1) . "\n";
2087             next;
2088         }
2089
2090         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2091             $descr = '' if !defined($descr);
2092             $descr .= PVE::Tools::decode_text($2);
2093         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2094             $conf->{snapstate} = $1;
2095         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2096             my $key = $1;
2097             my $value = $2;
2098             $conf->{$key} = $value;
2099         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2100             my $value = $1;
2101             if ($section eq 'pending') {
2102                 $conf->{delete} = $value; # we parse this later
2103             } else {
2104                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2105             }
2106         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2107             my $key = $1;
2108             my $value = $2;
2109             eval { $value = check_type($key, $value); };
2110             if ($@) {
2111                 warn "vm $vmid - unable to parse value of '$key' - $@";
2112             } else {
2113                 my $fmt = $confdesc->{$key}->{format};
2114                 if ($fmt && $fmt eq 'pve-qm-drive') {
2115                     my $v = parse_drive($key, $value);
2116                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2117                         $v->{file} = $volid;
2118                         $value = print_drive($vmid, $v);
2119                     } else {
2120                         warn "vm $vmid - unable to parse value of '$key'\n";
2121                         next;
2122                     }
2123                 }
2124
2125                 if ($key eq 'cdrom') {
2126                     $conf->{ide2} = $value;
2127                 } else {
2128                     $conf->{$key} = $value;
2129                 }
2130             }
2131         }
2132     }
2133
2134     if (defined($descr)) {
2135         $descr =~ s/\s+$//;
2136         $conf->{description} = $descr;
2137     }
2138     delete $res->{snapstate}; # just to be sure
2139
2140     return $res;
2141 }
2142
2143 sub write_vm_config {
2144     my ($filename, $conf) = @_;
2145
2146     delete $conf->{snapstate}; # just to be sure
2147
2148     if ($conf->{cdrom}) {
2149         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2150         $conf->{ide2} = $conf->{cdrom};
2151         delete $conf->{cdrom};
2152     }
2153
2154     # we do not use 'smp' any longer
2155     if ($conf->{sockets}) {
2156         delete $conf->{smp};
2157     } elsif ($conf->{smp}) {
2158         $conf->{sockets} = $conf->{smp};
2159         delete $conf->{cores};
2160         delete $conf->{smp};
2161     }
2162
2163     my $used_volids = {};
2164
2165     my $cleanup_config = sub {
2166         my ($cref, $pending, $snapname) = @_;
2167
2168         foreach my $key (keys %$cref) {
2169             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2170                 $key eq 'snapstate' || $key eq 'pending';
2171             my $value = $cref->{$key};
2172             if ($key eq 'delete') {
2173                 die "propertry 'delete' is only allowed in [PENDING]\n"
2174                     if !$pending;
2175                 # fixme: check syntax?
2176                 next;
2177             }
2178             eval { $value = check_type($key, $value); };
2179             die "unable to parse value of '$key' - $@" if $@;
2180
2181             $cref->{$key} = $value;
2182
2183             if (!$snapname && is_valid_drivename($key)) {
2184                 my $drive = parse_drive($key, $value);
2185                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2186             }
2187         }
2188     };
2189
2190     &$cleanup_config($conf);
2191
2192     &$cleanup_config($conf->{pending}, 1);
2193
2194     foreach my $snapname (keys %{$conf->{snapshots}}) {
2195         die "internal error" if $snapname eq 'pending';
2196         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2197     }
2198
2199     # remove 'unusedX' settings if we re-add a volume
2200     foreach my $key (keys %$conf) {
2201         my $value = $conf->{$key};
2202         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2203             delete $conf->{$key};
2204         }
2205     }
2206
2207     my $generate_raw_config = sub {
2208         my ($conf, $pending) = @_;
2209
2210         my $raw = '';
2211
2212         # add description as comment to top of file
2213         if (defined(my $descr = $conf->{description})) {
2214             if ($descr) {
2215                 foreach my $cl (split(/\n/, $descr)) {
2216                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2217                 }
2218             } else {
2219                 $raw .= "#\n" if $pending;
2220             }
2221         }
2222
2223         foreach my $key (sort keys %$conf) {
2224             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2225             $raw .= "$key: $conf->{$key}\n";
2226         }
2227         return $raw;
2228     };
2229
2230     my $raw = &$generate_raw_config($conf);
2231
2232     if (scalar(keys %{$conf->{pending}})){
2233         $raw .= "\n[PENDING]\n";
2234         $raw .= &$generate_raw_config($conf->{pending}, 1);
2235     }
2236
2237     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2238         $raw .= "\n[$snapname]\n";
2239         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2240     }
2241
2242     return $raw;
2243 }
2244
2245 sub load_defaults {
2246
2247     my $res = {};
2248
2249     # we use static defaults from our JSON schema configuration
2250     foreach my $key (keys %$confdesc) {
2251         if (defined(my $default = $confdesc->{$key}->{default})) {
2252             $res->{$key} = $default;
2253         }
2254     }
2255
2256     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2257     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2258
2259     return $res;
2260 }
2261
2262 sub config_list {
2263     my $vmlist = PVE::Cluster::get_vmlist();
2264     my $res = {};
2265     return $res if !$vmlist || !$vmlist->{ids};
2266     my $ids = $vmlist->{ids};
2267
2268     foreach my $vmid (keys %$ids) {
2269         my $d = $ids->{$vmid};
2270         next if !$d->{node} || $d->{node} ne $nodename;
2271         next if !$d->{type} || $d->{type} ne 'qemu';
2272         $res->{$vmid}->{exists} = 1;
2273     }
2274     return $res;
2275 }
2276
2277 # test if VM uses local resources (to prevent migration)
2278 sub check_local_resources {
2279     my ($conf, $noerr) = @_;
2280
2281     my $loc_res = 0;
2282
2283     $loc_res = 1 if $conf->{hostusb}; # old syntax
2284     $loc_res = 1 if $conf->{hostpci}; # old syntax
2285
2286     foreach my $k (keys %$conf) {
2287         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2288         # sockets are safe: they will recreated be on the target side post-migrate
2289         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2290         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2291     }
2292
2293     die "VM uses local resources\n" if $loc_res && !$noerr;
2294
2295     return $loc_res;
2296 }
2297
2298 # check if used storages are available on all nodes (use by migrate)
2299 sub check_storage_availability {
2300     my ($storecfg, $conf, $node) = @_;
2301
2302     foreach_drive($conf, sub {
2303         my ($ds, $drive) = @_;
2304
2305         my $volid = $drive->{file};
2306         return if !$volid;
2307
2308         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2309         return if !$sid;
2310
2311         # check if storage is available on both nodes
2312         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2313         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2314    });
2315 }
2316
2317 # list nodes where all VM images are available (used by has_feature API)
2318 sub shared_nodes {
2319     my ($conf, $storecfg) = @_;
2320
2321     my $nodelist = PVE::Cluster::get_nodelist();
2322     my $nodehash = { map { $_ => 1 } @$nodelist };
2323     my $nodename = PVE::INotify::nodename();
2324
2325     foreach_drive($conf, sub {
2326         my ($ds, $drive) = @_;
2327
2328         my $volid = $drive->{file};
2329         return if !$volid;
2330
2331         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2332         if ($storeid) {
2333             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2334             if ($scfg->{disable}) {
2335                 $nodehash = {};
2336             } elsif (my $avail = $scfg->{nodes}) {
2337                 foreach my $node (keys %$nodehash) {
2338                     delete $nodehash->{$node} if !$avail->{$node};
2339                 }
2340             } elsif (!$scfg->{shared}) {
2341                 foreach my $node (keys %$nodehash) {
2342                     delete $nodehash->{$node} if $node ne $nodename
2343                 }
2344             }
2345         }
2346     });
2347
2348     return $nodehash
2349 }
2350
2351 sub check_cmdline {
2352     my ($pidfile, $pid) = @_;
2353
2354     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2355     if (defined($fh)) {
2356         my $line = <$fh>;
2357         $fh->close;
2358         return undef if !$line;
2359         my @param = split(/\0/, $line);
2360
2361         my $cmd = $param[0];
2362         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2363
2364         for (my $i = 0; $i < scalar (@param); $i++) {
2365             my $p = $param[$i];
2366             next if !$p;
2367             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2368                 my $p = $param[$i+1];
2369                 return 1 if $p && ($p eq $pidfile);
2370                 return undef;
2371             }
2372         }
2373     }
2374     return undef;
2375 }
2376
2377 sub check_running {
2378     my ($vmid, $nocheck, $node) = @_;
2379
2380     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2381
2382     die "unable to find configuration file for VM $vmid - no such machine\n"
2383         if !$nocheck && ! -f $filename;
2384
2385     my $pidfile = pidfile_name($vmid);
2386
2387     if (my $fd = IO::File->new("<$pidfile")) {
2388         my $st = stat($fd);
2389         my $line = <$fd>;
2390         close($fd);
2391
2392         my $mtime = $st->mtime;
2393         if ($mtime > time()) {
2394             warn "file '$filename' modified in future\n";
2395         }
2396
2397         if ($line =~ m/^(\d+)$/) {
2398             my $pid = $1;
2399             if (check_cmdline($pidfile, $pid)) {
2400                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2401                     return $pid;
2402                 }
2403             }
2404         }
2405     }
2406
2407     return undef;
2408 }
2409
2410 sub vzlist {
2411
2412     my $vzlist = config_list();
2413
2414     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2415
2416     while (defined(my $de = $fd->read)) {
2417         next if $de !~ m/^(\d+)\.pid$/;
2418         my $vmid = $1;
2419         next if !defined($vzlist->{$vmid});
2420         if (my $pid = check_running($vmid)) {
2421             $vzlist->{$vmid}->{pid} = $pid;
2422         }
2423     }
2424
2425     return $vzlist;
2426 }
2427
2428 sub disksize {
2429     my ($storecfg, $conf) = @_;
2430
2431     my $bootdisk = $conf->{bootdisk};
2432     return undef if !$bootdisk;
2433     return undef if !is_valid_drivename($bootdisk);
2434
2435     return undef if !$conf->{$bootdisk};
2436
2437     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2438     return undef if !defined($drive);
2439
2440     return undef if drive_is_cdrom($drive);
2441
2442     my $volid = $drive->{file};
2443     return undef if !$volid;
2444
2445     return $drive->{size};
2446 }
2447
2448 my $last_proc_pid_stat;
2449
2450 # get VM status information
2451 # This must be fast and should not block ($full == false)
2452 # We only query KVM using QMP if $full == true (this can be slow)
2453 sub vmstatus {
2454     my ($opt_vmid, $full) = @_;
2455
2456     my $res = {};
2457
2458     my $storecfg = PVE::Storage::config();
2459
2460     my $list = vzlist();
2461     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2462
2463     my $cpucount = $cpuinfo->{cpus} || 1;
2464
2465     foreach my $vmid (keys %$list) {
2466         next if $opt_vmid && ($vmid ne $opt_vmid);
2467
2468         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2469         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2470
2471         my $d = {};
2472         $d->{pid} = $list->{$vmid}->{pid};
2473
2474         # fixme: better status?
2475         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2476
2477         my $size = disksize($storecfg, $conf);
2478         if (defined($size)) {
2479             $d->{disk} = 0; # no info available
2480             $d->{maxdisk} = $size;
2481         } else {
2482             $d->{disk} = 0;
2483             $d->{maxdisk} = 0;
2484         }
2485
2486         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2487         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2488         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2489
2490         $d->{name} = $conf->{name} || "VM $vmid";
2491         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2492
2493         if ($conf->{balloon}) {
2494             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2495             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2496         }
2497
2498         $d->{uptime} = 0;
2499         $d->{cpu} = 0;
2500         $d->{mem} = 0;
2501
2502         $d->{netout} = 0;
2503         $d->{netin} = 0;
2504
2505         $d->{diskread} = 0;
2506         $d->{diskwrite} = 0;
2507
2508         $d->{template} = PVE::QemuConfig->is_template($conf);
2509
2510         $res->{$vmid} = $d;
2511     }
2512
2513     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2514     foreach my $dev (keys %$netdev) {
2515         next if $dev !~ m/^tap([1-9]\d*)i/;
2516         my $vmid = $1;
2517         my $d = $res->{$vmid};
2518         next if !$d;
2519
2520         $d->{netout} += $netdev->{$dev}->{receive};
2521         $d->{netin} += $netdev->{$dev}->{transmit};
2522
2523         if ($full) {
2524             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2525             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2526         }
2527
2528     }
2529
2530     my $ctime = gettimeofday;
2531
2532     foreach my $vmid (keys %$list) {
2533
2534         my $d = $res->{$vmid};
2535         my $pid = $d->{pid};
2536         next if !$pid;
2537
2538         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2539         next if !$pstat; # not running
2540
2541         my $used = $pstat->{utime} + $pstat->{stime};
2542
2543         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2544
2545         if ($pstat->{vsize}) {
2546             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2547         }
2548
2549         my $old = $last_proc_pid_stat->{$pid};
2550         if (!$old) {
2551             $last_proc_pid_stat->{$pid} = {
2552                 time => $ctime,
2553                 used => $used,
2554                 cpu => 0,
2555             };
2556             next;
2557         }
2558
2559         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2560
2561         if ($dtime > 1000) {
2562             my $dutime = $used -  $old->{used};
2563
2564             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2565             $last_proc_pid_stat->{$pid} = {
2566                 time => $ctime,
2567                 used => $used,
2568                 cpu => $d->{cpu},
2569             };
2570         } else {
2571             $d->{cpu} = $old->{cpu};
2572         }
2573     }
2574
2575     return $res if !$full;
2576
2577     my $qmpclient = PVE::QMPClient->new();
2578
2579     my $ballooncb = sub {
2580         my ($vmid, $resp) = @_;
2581
2582         my $info = $resp->{'return'};
2583         return if !$info->{max_mem};
2584
2585         my $d = $res->{$vmid};
2586
2587         # use memory assigned to VM
2588         $d->{maxmem} = $info->{max_mem};
2589         $d->{balloon} = $info->{actual};
2590
2591         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2592             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2593             $d->{freemem} = $info->{free_mem};
2594         }
2595
2596         $d->{ballooninfo} = $info;
2597     };
2598
2599     my $blockstatscb = sub {
2600         my ($vmid, $resp) = @_;
2601         my $data = $resp->{'return'} || [];
2602         my $totalrdbytes = 0;
2603         my $totalwrbytes = 0;
2604
2605         for my $blockstat (@$data) {
2606             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2607             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2608
2609             $blockstat->{device} =~ s/drive-//;
2610             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2611         }
2612         $res->{$vmid}->{diskread} = $totalrdbytes;
2613         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2614     };
2615
2616     my $statuscb = sub {
2617         my ($vmid, $resp) = @_;
2618
2619         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2620         # this fails if ballon driver is not loaded, so this must be
2621         # the last commnand (following command are aborted if this fails).
2622         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2623
2624         my $status = 'unknown';
2625         if (!defined($status = $resp->{'return'}->{status})) {
2626             warn "unable to get VM status\n";
2627             return;
2628         }
2629
2630         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2631     };
2632
2633     foreach my $vmid (keys %$list) {
2634         next if $opt_vmid && ($vmid ne $opt_vmid);
2635         next if !$res->{$vmid}->{pid}; # not running
2636         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2637     }
2638
2639     $qmpclient->queue_execute(undef, 1);
2640
2641     foreach my $vmid (keys %$list) {
2642         next if $opt_vmid && ($vmid ne $opt_vmid);
2643         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2644     }
2645
2646     return $res;
2647 }
2648
2649 sub foreach_drive {
2650     my ($conf, $func) = @_;
2651
2652     foreach my $ds (valid_drive_names()) {
2653         next if !defined($conf->{$ds});
2654
2655         my $drive = parse_drive($ds, $conf->{$ds});
2656         next if !$drive;
2657
2658         &$func($ds, $drive);
2659     }
2660 }
2661
2662 sub foreach_volid {
2663     my ($conf, $func) = @_;
2664
2665     my $volhash = {};
2666
2667     my $test_volid = sub {
2668         my ($volid, $is_cdrom) = @_;
2669
2670         return if !$volid;
2671
2672         $volhash->{$volid} = $is_cdrom || 0;
2673     };
2674
2675     foreach_drive($conf, sub {
2676         my ($ds, $drive) = @_;
2677         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2678     });
2679
2680     foreach my $snapname (keys %{$conf->{snapshots}}) {
2681         my $snap = $conf->{snapshots}->{$snapname};
2682         &$test_volid($snap->{vmstate}, 0);
2683         foreach_drive($snap, sub {
2684             my ($ds, $drive) = @_;
2685             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2686         });
2687     }
2688
2689     foreach my $volid (keys %$volhash) {
2690         &$func($volid, $volhash->{$volid});
2691     }
2692 }
2693
2694 sub vga_conf_has_spice {
2695     my ($vga) = @_;
2696
2697     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2698
2699     return $1 || 1;
2700 }
2701
2702 sub config_to_command {
2703     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2704
2705     my $cmd = [];
2706     my $globalFlags = [];
2707     my $machineFlags = [];
2708     my $rtcFlags = [];
2709     my $cpuFlags = [];
2710     my $devices = [];
2711     my $pciaddr = '';
2712     my $bridges = {};
2713     my $kvmver = kvm_user_version();
2714     my $vernum = 0; # unknown
2715     my $ostype = $conf->{ostype};
2716     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2717         $vernum = $1*1000000+$2*1000;
2718     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2719         $vernum = $1*1000000+$2*1000+$3;
2720     }
2721
2722     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2723
2724     my $have_ovz = -f '/proc/vz/vestat';
2725
2726     my $q35 = machine_type_is_q35($conf);
2727     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2728     my $machine_type = $forcemachine || $conf->{machine};
2729     my $use_old_bios_files = undef;
2730     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2731
2732     my $cpuunits = defined($conf->{cpuunits}) ?
2733             $conf->{cpuunits} : $defaults->{cpuunits};
2734
2735     push @$cmd, '/usr/bin/systemd-run';
2736     push @$cmd, '--scope';
2737     push @$cmd, '--slice', "qemu";
2738     push @$cmd, '--unit', $vmid;
2739     push @$cmd, '--description', "'Proxmox VE VM $vmid'";
2740     # set KillMode=none, so that systemd don't kill those scopes
2741     # at shutdown (pve-manager service should stop the VMs instead)
2742     push @$cmd, '-p', "KillMode=none";
2743     push @$cmd, '-p', "CPUShares=$cpuunits";
2744     if ($conf->{cpulimit}) {
2745         my $cpulimit = int($conf->{cpulimit} * 100);
2746         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2747     }
2748
2749     push @$cmd, '/usr/bin/kvm';
2750
2751     push @$cmd, '-id', $vmid;
2752
2753     my $use_virtio = 0;
2754
2755     my $qmpsocket = qmp_socket($vmid);
2756     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2757     push @$cmd, '-mon', "chardev=qmp,mode=control";
2758
2759
2760     push @$cmd, '-pidfile' , pidfile_name($vmid);
2761
2762     push @$cmd, '-daemonize';
2763
2764     if ($conf->{smbios1}) {
2765         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2766     }
2767
2768     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2769         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2770         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2771         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2772         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2773         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2774         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2775     }
2776
2777     if ($q35) {
2778         # the q35 chipset support native usb2, so we enable usb controller
2779         # by default for this machine type
2780         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2781     } else {
2782         $pciaddr = print_pci_addr("piix3", $bridges);
2783         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2784
2785         my $use_usb2 = 0;
2786         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2787             next if !$conf->{"usb$i"};
2788             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2789             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2790             $use_usb2 = 1;
2791         }
2792         # include usb device config
2793         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2794     }
2795
2796     # add usb3 controller if needed
2797
2798     my $use_usb3 = 0;
2799     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2800         next if !$conf->{"usb$i"};
2801         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2802         next if !$d || !$d->{usb3};
2803         $use_usb3 = 1;
2804     }
2805
2806     $pciaddr = print_pci_addr("xhci", $bridges);
2807     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2808
2809     my $vga = $conf->{vga};
2810
2811     my $qxlnum = vga_conf_has_spice($vga);
2812     $vga = 'qxl' if $qxlnum;
2813
2814     if (!$vga) {
2815         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2816                                 $conf->{ostype} eq 'win7' ||
2817                                 $conf->{ostype} eq 'w2k8')) {
2818             $vga = 'std';
2819         } else {
2820             $vga = 'cirrus';
2821         }
2822     }
2823
2824     # enable absolute mouse coordinates (needed by vnc)
2825     my $tablet;
2826     if (defined($conf->{tablet})) {
2827         $tablet = $conf->{tablet};
2828     } else {
2829         $tablet = $defaults->{tablet};
2830         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2831         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2832     }
2833
2834     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2835
2836     my $kvm_off = 0;
2837     # host pci devices
2838     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2839         my $d = parse_hostpci($conf->{"hostpci$i"});
2840         next if !$d;
2841
2842         my $pcie = $d->{pcie};
2843         if($pcie){
2844             die "q35 machine model is not enabled" if !$q35;
2845             $pciaddr = print_pcie_addr("hostpci$i");
2846         }else{
2847             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2848         }
2849
2850         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2851         my $xvga = '';
2852         if ($d->{'x-vga'}) {
2853             $xvga = ',x-vga=on';
2854             $kvm_off = 1;
2855             $vga = 'none';
2856             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2857                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2858             }
2859             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2860                 $xvga = "";
2861             }
2862         }
2863         my $pcidevices = $d->{pciid};
2864         my $multifunction = 1 if @$pcidevices > 1;
2865
2866         my $j=0;
2867         foreach my $pcidevice (@$pcidevices) {
2868
2869             my $id = "hostpci$i";
2870             $id .= ".$j" if $multifunction;
2871             my $addr = $pciaddr;
2872             $addr .= ".$j" if $multifunction;
2873             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2874
2875             if($j == 0){
2876                 $devicestr .= "$rombar$xvga";
2877                 $devicestr .= ",multifunction=on" if $multifunction;
2878             }
2879
2880             push @$devices, '-device', $devicestr;
2881             $j++;
2882         }
2883     }
2884
2885     # usb devices
2886     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2887         next if !$conf->{"usb$i"};
2888         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2889         next if !$d;
2890
2891         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2892         my $usbbus = '';
2893         if (defined($d->{usb3}) && $d->{usb3}) {
2894             $usbbus = ',bus=xhci.0';
2895         }
2896
2897         if (defined($d->{host})) {
2898             $d = parse_usb_device($d->{host});
2899             if (defined($d->{vendorid}) && defined($d->{productid})) {
2900                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2901             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2902                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2903             } elsif (defined($d->{spice}) && $d->{spice}) {
2904                 # usb redir support for spice, currently no usb3
2905                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2906                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2907             }
2908         }
2909     }
2910
2911     # serial devices
2912     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2913         if (my $path = $conf->{"serial$i"}) {
2914             if ($path eq 'socket') {
2915                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2916                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2917                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2918             } else {
2919                 die "no such serial device\n" if ! -c $path;
2920                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2921                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2922             }
2923         }
2924     }
2925
2926     # parallel devices
2927     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2928         if (my $path = $conf->{"parallel$i"}) {
2929             die "no such parallel device\n" if ! -c $path;
2930             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2931             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2932             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2933         }
2934     }
2935
2936     my $vmname = $conf->{name} || "vm$vmid";
2937
2938     push @$cmd, '-name', $vmname;
2939
2940     my $sockets = 1;
2941     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2942     $sockets = $conf->{sockets} if  $conf->{sockets};
2943
2944     my $cores = $conf->{cores} || 1;
2945
2946     my $maxcpus = $sockets * $cores;
2947
2948     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2949
2950     my $allowed_vcpus = $cpuinfo->{cpus};
2951
2952     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2953         if ($allowed_vcpus < $maxcpus);
2954
2955     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2956
2957     push @$cmd, '-nodefaults';
2958
2959     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2960
2961     my $bootindex_hash = {};
2962     my $i = 1;
2963     foreach my $o (split(//, $bootorder)) {
2964         $bootindex_hash->{$o} = $i*100;
2965         $i++;
2966     }
2967
2968     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2969
2970     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2971
2972     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2973
2974     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2975
2976     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2977         my $socket = vnc_socket($vmid);
2978         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2979     } else {
2980         push @$cmd, '-nographic';
2981     }
2982
2983     # time drift fix
2984     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2985
2986     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2987     my $useLocaltime = $conf->{localtime};
2988
2989     if ($ostype) {
2990         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2991
2992         if ($ostype =~ m/^w/) { # windows
2993             $useLocaltime = 1 if !defined($conf->{localtime});
2994
2995             # use time drift fix when acpi is enabled
2996             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2997                 $tdf = 1 if !defined($conf->{tdf});
2998             }
2999         }
3000
3001         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3002             $ostype eq 'wvista') {
3003             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3004             push @$cmd, '-no-hpet';
3005             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3006                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3007                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3008                 push @$cpuFlags , 'hv_time' if !$nokvm;
3009
3010             } else {
3011                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3012             }
3013         }
3014
3015         if ($ostype eq 'win7' || $ostype eq 'win8') {
3016             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3017         }
3018     }
3019
3020     push @$rtcFlags, 'driftfix=slew' if $tdf;
3021
3022     if ($nokvm) {
3023         push @$machineFlags, 'accel=tcg';
3024     } else {
3025         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3026     }
3027
3028     if ($machine_type) {
3029         push @$machineFlags, "type=${machine_type}";
3030     }
3031
3032     if ($conf->{startdate}) {
3033         push @$rtcFlags, "base=$conf->{startdate}";
3034     } elsif ($useLocaltime) {
3035         push @$rtcFlags, 'base=localtime';
3036     }
3037
3038     my $cpu = $nokvm ? "qemu64" : "kvm64";
3039     if (my $cputype = $conf->{cpu}) {
3040         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3041             or die "Cannot parse cpu description: $cputype\n";
3042         $cpu = $cpuconf->{cputype};
3043         $kvm_off = 1 if $cpuconf->{hidden};
3044     }
3045
3046     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3047
3048     push @$cpuFlags , '-x2apic'
3049         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3050
3051     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3052
3053     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3054
3055     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3056
3057         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3058         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3059     }
3060
3061     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3062
3063     push @$cpuFlags, 'kvm=off' if $kvm_off;
3064
3065     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3066         die "internal error"; # should not happen
3067
3068     push @$cpuFlags, "vendor=${cpu_vendor}"
3069         if $cpu_vendor ne 'default';
3070
3071     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3072
3073     push @$cmd, '-cpu', $cpu;
3074
3075     my $memory = $conf->{memory} || $defaults->{memory};
3076     my $static_memory = 0;
3077     my $dimm_memory = 0;
3078
3079     if ($hotplug_features->{memory}) {
3080         die "NUMA need to be enabled for memory hotplug\n" if !$conf->{numa};
3081         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3082         $static_memory = $STATICMEM;
3083         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3084         $dimm_memory = $memory - $static_memory;
3085         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3086
3087     } else {
3088
3089         $static_memory = $memory;
3090         push @$cmd, '-m', $static_memory;
3091     }
3092
3093     if ($conf->{numa}) {
3094
3095         my $numa_totalmemory = undef;
3096         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3097             next if !$conf->{"numa$i"};
3098             my $numa = parse_numa($conf->{"numa$i"});
3099             next if !$numa;
3100             # memory
3101             die "missing NUMA node$i memory value\n" if !$numa->{memory};
3102             my $numa_memory = $numa->{memory};
3103             $numa_totalmemory += $numa_memory;
3104             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3105
3106             # cpus
3107             my $cpulists = $numa->{cpus};
3108             die "missing NUMA node$i cpus\n" if !defined($cpulists);
3109             my $cpus = join(',', map {
3110                 my ($start, $end) = @$_;
3111                 defined($end) ? "$start-$end" : $start
3112             } @$cpulists);
3113
3114             # hostnodes
3115             my $hostnodelists = $numa->{hostnodes};
3116             if (defined($hostnodelists)) {
3117                 my $hostnodes;
3118                 foreach my $hostnoderange (@$hostnodelists) {
3119                     my ($start, $end) = @$hostnoderange;
3120                     $hostnodes .= ',' if $hostnodes;
3121                     $hostnodes .= $start;
3122                     $hostnodes .= "-$end" if defined($end);
3123                     $end //= $start;
3124                     for (my $i = $start; $i <= $end; ++$i ) {
3125                         die "host NUMA node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3126                     }
3127                 }
3128
3129                 # policy
3130                 my $policy = $numa->{policy};
3131                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3132                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3133             }
3134
3135             push @$cmd, '-object', $numa_object;
3136             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3137         }
3138
3139         die "total memory for NUMA nodes must be equal to vm static memory\n"
3140             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3141
3142         #if no custom tology, we split memory and cores across numa nodes
3143         if(!$numa_totalmemory) {
3144
3145             my $numa_memory = ($static_memory / $sockets) . "M";
3146
3147             for (my $i = 0; $i < $sockets; $i++)  {
3148
3149                 my $cpustart = ($cores * $i);
3150                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3151                 my $cpus = $cpustart;
3152                 $cpus .= "-$cpuend" if $cpuend;
3153
3154                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3155                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3156             }
3157         }
3158     }
3159
3160     if ($hotplug_features->{memory}) {
3161         PVE::QemuServer::Memory::foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3162             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3163             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3164             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3165
3166             #if dimm_memory is not aligned to dimm map
3167             if($current_size > $memory) {
3168                  $conf->{memory} = $current_size;
3169                  PVE::QemuConfig->write_config($vmid, $conf);
3170             }
3171         });
3172     }
3173
3174     push @$cmd, '-S' if $conf->{freeze};
3175
3176     # set keyboard layout
3177     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3178     push @$cmd, '-k', $kb if $kb;
3179
3180     # enable sound
3181     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3182     #push @$cmd, '-soundhw', 'es1370';
3183     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3184
3185     if($conf->{agent}) {
3186         my $qgasocket = qmp_socket($vmid, 1);
3187         my $pciaddr = print_pci_addr("qga0", $bridges);
3188         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3189         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3190         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3191     }
3192
3193     my $spice_port;
3194
3195     if ($qxlnum) {
3196         if ($qxlnum > 1) {
3197             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3198                 for(my $i = 1; $i < $qxlnum; $i++){
3199                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3200                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3201                 }
3202             } else {
3203                 # assume other OS works like Linux
3204                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3205                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3206             }
3207         }
3208
3209         my $pciaddr = print_pci_addr("spice", $bridges);
3210
3211         my $nodename = PVE::INotify::nodename();
3212         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3213         $spice_port = PVE::Tools::next_spice_port($pfamily);
3214
3215         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3216
3217         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3218         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3219         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3220     }
3221
3222     # enable balloon by default, unless explicitly disabled
3223     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3224         $pciaddr = print_pci_addr("balloon0", $bridges);
3225         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3226     }
3227
3228     if ($conf->{watchdog}) {
3229         my $wdopts = parse_watchdog($conf->{watchdog});
3230         $pciaddr = print_pci_addr("watchdog", $bridges);
3231         my $watchdog = $wdopts->{model} || 'i6300esb';
3232         push @$devices, '-device', "$watchdog$pciaddr";
3233         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3234     }
3235
3236     my $vollist = [];
3237     my $scsicontroller = {};
3238     my $ahcicontroller = {};
3239     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3240
3241     # Add iscsi initiator name if available
3242     if (my $initiator = get_initiator_name()) {
3243         push @$devices, '-iscsi', "initiator-name=$initiator";
3244     }
3245
3246     foreach_drive($conf, sub {
3247         my ($ds, $drive) = @_;
3248
3249         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3250             push @$vollist, $drive->{file};
3251         }
3252
3253         $use_virtio = 1 if $ds =~ m/^virtio/;
3254
3255         if (drive_is_cdrom ($drive)) {
3256             if ($bootindex_hash->{d}) {
3257                 $drive->{bootindex} = $bootindex_hash->{d};
3258                 $bootindex_hash->{d} += 1;
3259             }
3260         } else {
3261             if ($bootindex_hash->{c}) {
3262                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3263                 $bootindex_hash->{c} += 1;
3264             }
3265         }
3266
3267         if($drive->{interface} eq 'virtio'){
3268            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3269         }
3270
3271         if ($drive->{interface} eq 'scsi') {
3272
3273             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3274
3275             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3276             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3277
3278             my $iothread = '';
3279             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3280                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3281                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3282             } elsif ($drive->{iothread}) {
3283                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3284             }
3285
3286             my $queues = '';
3287             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3288                 $queues = ",num_queues=$drive->{queues}";
3289             }
3290
3291             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3292             $scsicontroller->{$controller}=1;
3293         }
3294
3295         if ($drive->{interface} eq 'sata') {
3296            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3297            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3298            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3299            $ahcicontroller->{$controller}=1;
3300         }
3301
3302         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3303         push @$devices, '-drive',$drive_cmd;
3304         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3305     });
3306
3307     for (my $i = 0; $i < $MAX_NETS; $i++) {
3308          next if !$conf->{"net$i"};
3309          my $d = parse_net($conf->{"net$i"});
3310          next if !$d;
3311
3312          $use_virtio = 1 if $d->{model} eq 'virtio';
3313
3314          if ($bootindex_hash->{n}) {
3315             $d->{bootindex} = $bootindex_hash->{n};
3316             $bootindex_hash->{n} += 1;
3317          }
3318
3319          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3320          push @$devices, '-netdev', $netdevfull;
3321
3322          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3323          push @$devices, '-device', $netdevicefull;
3324     }
3325
3326     if (!$q35) {
3327         # add pci bridges
3328         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3329            $bridges->{1} = 1;
3330            $bridges->{2} = 1;
3331         }
3332
3333         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3334
3335         while (my ($k, $v) = each %$bridges) {
3336             $pciaddr = print_pci_addr("pci.$k");
3337             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3338         }
3339     }
3340
3341     # add custom args
3342     if ($conf->{args}) {
3343         my $aa = PVE::Tools::split_args($conf->{args});
3344         push @$cmd, @$aa;
3345     }
3346
3347     push @$cmd, @$devices;
3348     push @$cmd, '-rtc', join(',', @$rtcFlags)
3349         if scalar(@$rtcFlags);
3350     push @$cmd, '-machine', join(',', @$machineFlags)
3351         if scalar(@$machineFlags);
3352     push @$cmd, '-global', join(',', @$globalFlags)
3353         if scalar(@$globalFlags);
3354
3355     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3356 }
3357
3358 sub vnc_socket {
3359     my ($vmid) = @_;
3360     return "${var_run_tmpdir}/$vmid.vnc";
3361 }
3362
3363 sub spice_port {
3364     my ($vmid) = @_;
3365
3366     my $res = vm_mon_cmd($vmid, 'query-spice');
3367
3368     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3369 }
3370
3371 sub qmp_socket {
3372     my ($vmid, $qga) = @_;
3373     my $sockettype = $qga ? 'qga' : 'qmp';
3374     return "${var_run_tmpdir}/$vmid.$sockettype";
3375 }
3376
3377 sub pidfile_name {
3378     my ($vmid) = @_;
3379     return "${var_run_tmpdir}/$vmid.pid";
3380 }
3381
3382 sub vm_devices_list {
3383     my ($vmid) = @_;
3384
3385     my $res = vm_mon_cmd($vmid, 'query-pci');
3386     my $devices = {};
3387     foreach my $pcibus (@$res) {
3388         foreach my $device (@{$pcibus->{devices}}) {
3389             next if !$device->{'qdev_id'};
3390             if ($device->{'pci_bridge'}) {
3391                 $devices->{$device->{'qdev_id'}} = 1;
3392                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3393                     next if !$bridge_device->{'qdev_id'};
3394                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3395                     $devices->{$device->{'qdev_id'}}++;
3396                 }
3397             } else {
3398                 $devices->{$device->{'qdev_id'}} = 1;
3399             }
3400         }
3401     }
3402
3403     my $resblock = vm_mon_cmd($vmid, 'query-block');
3404     foreach my $block (@$resblock) {
3405         if($block->{device} =~ m/^drive-(\S+)/){
3406                 $devices->{$1} = 1;
3407         }
3408     }
3409
3410     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3411     foreach my $mice (@$resmice) {
3412         if ($mice->{name} eq 'QEMU HID Tablet') {
3413             $devices->{tablet} = 1;
3414             last;
3415         }
3416     }
3417
3418     return $devices;
3419 }
3420
3421 sub vm_deviceplug {
3422     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3423
3424     my $q35 = machine_type_is_q35($conf);
3425
3426     my $devices_list = vm_devices_list($vmid);
3427     return 1 if defined($devices_list->{$deviceid});
3428
3429     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3430
3431     if ($deviceid eq 'tablet') {
3432
3433         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3434
3435     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3436
3437         qemu_iothread_add($vmid, $deviceid, $device);
3438
3439         qemu_driveadd($storecfg, $vmid, $device);
3440         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3441
3442         qemu_deviceadd($vmid, $devicefull);
3443         eval { qemu_deviceaddverify($vmid, $deviceid); };
3444         if (my $err = $@) {
3445             eval { qemu_drivedel($vmid, $deviceid); };
3446             warn $@ if $@;
3447             die $err;
3448         }
3449
3450     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3451
3452
3453         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3454         my $pciaddr = print_pci_addr($deviceid);
3455         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3456
3457         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3458
3459         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3460             qemu_iothread_add($vmid, $deviceid, $device);
3461             $devicefull .= ",iothread=iothread-$deviceid";
3462         }
3463
3464         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3465             $devicefull .= ",num_queues=$device->{queues}";
3466         }
3467
3468         qemu_deviceadd($vmid, $devicefull);
3469         qemu_deviceaddverify($vmid, $deviceid);
3470
3471     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3472
3473         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3474         qemu_driveadd($storecfg, $vmid, $device);
3475
3476         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3477         eval { qemu_deviceadd($vmid, $devicefull); };
3478         if (my $err = $@) {
3479             eval { qemu_drivedel($vmid, $deviceid); };
3480             warn $@ if $@;
3481             die $err;
3482         }
3483
3484     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3485
3486         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3487
3488         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3489         my $use_old_bios_files = undef;
3490         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3491
3492         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3493         qemu_deviceadd($vmid, $netdevicefull);
3494         eval { qemu_deviceaddverify($vmid, $deviceid); };
3495         if (my $err = $@) {
3496             eval { qemu_netdevdel($vmid, $deviceid); };
3497             warn $@ if $@;
3498             die $err;
3499         }
3500
3501     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3502
3503         my $bridgeid = $2;
3504         my $pciaddr = print_pci_addr($deviceid);
3505         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3506
3507         qemu_deviceadd($vmid, $devicefull);
3508         qemu_deviceaddverify($vmid, $deviceid);
3509
3510     } else {
3511         die "can't hotplug device '$deviceid'\n";
3512     }
3513
3514     return 1;
3515 }
3516
3517 # fixme: this should raise exceptions on error!
3518 sub vm_deviceunplug {
3519     my ($vmid, $conf, $deviceid) = @_;
3520
3521     my $devices_list = vm_devices_list($vmid);
3522     return 1 if !defined($devices_list->{$deviceid});
3523
3524     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3525
3526     if ($deviceid eq 'tablet') {
3527
3528         qemu_devicedel($vmid, $deviceid);
3529
3530     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3531
3532         qemu_devicedel($vmid, $deviceid);
3533         qemu_devicedelverify($vmid, $deviceid);
3534         qemu_drivedel($vmid, $deviceid);
3535         qemu_iothread_del($conf, $vmid, $deviceid);
3536
3537     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3538
3539         qemu_devicedel($vmid, $deviceid);
3540         qemu_devicedelverify($vmid, $deviceid);
3541         qemu_iothread_del($conf, $vmid, $deviceid);
3542
3543     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3544
3545         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3546         my $device = parse_drive($deviceid, $conf->{$deviceid});
3547         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3548
3549         qemu_devicedel($vmid, $deviceid);
3550         qemu_drivedel($vmid, $deviceid);
3551         qemu_deletescsihw($conf, $vmid, $deviceid);
3552
3553     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3554
3555         qemu_devicedel($vmid, $deviceid);
3556         qemu_devicedelverify($vmid, $deviceid);
3557         qemu_netdevdel($vmid, $deviceid);
3558
3559     } else {
3560         die "can't unplug device '$deviceid'\n";
3561     }
3562
3563     return 1;
3564 }
3565
3566 sub qemu_deviceadd {
3567     my ($vmid, $devicefull) = @_;
3568
3569     $devicefull = "driver=".$devicefull;
3570     my %options =  split(/[=,]/, $devicefull);
3571
3572     vm_mon_cmd($vmid, "device_add" , %options);
3573 }
3574
3575 sub qemu_devicedel {
3576     my ($vmid, $deviceid) = @_;
3577
3578     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3579 }
3580
3581 sub qemu_iothread_add {
3582     my($vmid, $deviceid, $device) = @_;
3583
3584     if ($device->{iothread}) {
3585         my $iothreads = vm_iothreads_list($vmid);
3586         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3587     }
3588 }
3589
3590 sub qemu_iothread_del {
3591     my($conf, $vmid, $deviceid) = @_;
3592
3593     my $device = parse_drive($deviceid, $conf->{$deviceid});
3594     if ($device->{iothread}) {
3595         my $iothreads = vm_iothreads_list($vmid);
3596         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3597     }
3598 }
3599
3600 sub qemu_objectadd {
3601     my($vmid, $objectid, $qomtype) = @_;
3602
3603     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3604
3605     return 1;
3606 }
3607
3608 sub qemu_objectdel {
3609     my($vmid, $objectid) = @_;
3610
3611     vm_mon_cmd($vmid, "object-del", id => $objectid);
3612
3613     return 1;
3614 }
3615
3616 sub qemu_driveadd {
3617     my ($storecfg, $vmid, $device) = @_;
3618
3619     my $drive = print_drive_full($storecfg, $vmid, $device);
3620     $drive =~ s/\\/\\\\/g;
3621     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3622
3623     # If the command succeeds qemu prints: "OK"
3624     return 1 if $ret =~ m/OK/s;
3625
3626     die "adding drive failed: $ret\n";
3627 }
3628
3629 sub qemu_drivedel {
3630     my($vmid, $deviceid) = @_;
3631
3632     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3633     $ret =~ s/^\s+//;
3634
3635     return 1 if $ret eq "";
3636
3637     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3638     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3639
3640     die "deleting drive $deviceid failed : $ret\n";
3641 }
3642
3643 sub qemu_deviceaddverify {
3644     my ($vmid, $deviceid) = @_;
3645
3646     for (my $i = 0; $i <= 5; $i++) {
3647          my $devices_list = vm_devices_list($vmid);
3648          return 1 if defined($devices_list->{$deviceid});
3649          sleep 1;
3650     }
3651
3652     die "error on hotplug device '$deviceid'\n";
3653 }
3654
3655
3656 sub qemu_devicedelverify {
3657     my ($vmid, $deviceid) = @_;
3658
3659     # need to verify that the device is correctly removed as device_del
3660     # is async and empty return is not reliable
3661
3662     for (my $i = 0; $i <= 5; $i++) {
3663          my $devices_list = vm_devices_list($vmid);
3664          return 1 if !defined($devices_list->{$deviceid});
3665          sleep 1;
3666     }
3667
3668     die "error on hot-unplugging device '$deviceid'\n";
3669 }
3670
3671 sub qemu_findorcreatescsihw {
3672     my ($storecfg, $conf, $vmid, $device) = @_;
3673
3674     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3675
3676     my $scsihwid="$controller_prefix$controller";
3677     my $devices_list = vm_devices_list($vmid);
3678
3679     if(!defined($devices_list->{$scsihwid})) {
3680         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3681     }
3682
3683     return 1;
3684 }
3685
3686 sub qemu_deletescsihw {
3687     my ($conf, $vmid, $opt) = @_;
3688
3689     my $device = parse_drive($opt, $conf->{$opt});
3690
3691     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3692         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3693         return 1;
3694     }
3695
3696     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3697
3698     my $devices_list = vm_devices_list($vmid);
3699     foreach my $opt (keys %{$devices_list}) {
3700         if (PVE::QemuServer::is_valid_drivename($opt)) {
3701             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3702             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3703                 return 1;
3704             }
3705         }
3706     }
3707
3708     my $scsihwid="scsihw$controller";
3709
3710     vm_deviceunplug($vmid, $conf, $scsihwid);
3711
3712     return 1;
3713 }
3714
3715 sub qemu_add_pci_bridge {
3716     my ($storecfg, $conf, $vmid, $device) = @_;
3717
3718     my $bridges = {};
3719
3720     my $bridgeid;
3721
3722     print_pci_addr($device, $bridges);
3723
3724     while (my ($k, $v) = each %$bridges) {
3725         $bridgeid = $k;
3726     }
3727     return 1 if !defined($bridgeid) || $bridgeid < 1;
3728
3729     my $bridge = "pci.$bridgeid";
3730     my $devices_list = vm_devices_list($vmid);
3731
3732     if (!defined($devices_list->{$bridge})) {
3733         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3734     }
3735
3736     return 1;
3737 }
3738
3739 sub qemu_set_link_status {
3740     my ($vmid, $device, $up) = @_;
3741
3742     vm_mon_cmd($vmid, "set_link", name => $device,
3743                up => $up ? JSON::true : JSON::false);
3744 }
3745
3746 sub qemu_netdevadd {
3747     my ($vmid, $conf, $device, $deviceid) = @_;
3748
3749     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3750     my %options =  split(/[=,]/, $netdev);
3751
3752     vm_mon_cmd($vmid, "netdev_add",  %options);
3753     return 1;
3754 }
3755
3756 sub qemu_netdevdel {
3757     my ($vmid, $deviceid) = @_;
3758
3759     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3760 }
3761
3762 sub qemu_cpu_hotplug {
3763     my ($vmid, $conf, $vcpus) = @_;
3764
3765     my $sockets = 1;
3766     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3767     $sockets = $conf->{sockets} if  $conf->{sockets};
3768     my $cores = $conf->{cores} || 1;
3769     my $maxcpus = $sockets * $cores;
3770
3771     $vcpus = $maxcpus if !$vcpus;
3772
3773     die "you can't add more vcpus than maxcpus\n"
3774         if $vcpus > $maxcpus;
3775
3776     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3777     die "online cpu unplug is not yet possible\n"
3778         if $vcpus < $currentvcpus;
3779
3780     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3781     die "vcpus in running vm is different than configuration\n"
3782         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3783
3784     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3785         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3786     }
3787 }
3788
3789 sub qemu_block_set_io_throttle {
3790     my ($vmid, $deviceid,
3791         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3792         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3793
3794     return if !check_running($vmid) ;
3795
3796     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3797         bps => int($bps),
3798         bps_rd => int($bps_rd),
3799         bps_wr => int($bps_wr),
3800         iops => int($iops),
3801         iops_rd => int($iops_rd),
3802         iops_wr => int($iops_wr),
3803         bps_max => int($bps_max),
3804         bps_rd_max => int($bps_rd_max),
3805         bps_wr_max => int($bps_wr_max),
3806         iops_max => int($iops_max),
3807         iops_rd_max => int($iops_rd_max),
3808         iops_wr_max => int($iops_wr_max)
3809     );
3810
3811 }
3812
3813 # old code, only used to shutdown old VM after update
3814 sub __read_avail {
3815     my ($fh, $timeout) = @_;
3816
3817     my $sel = new IO::Select;
3818     $sel->add($fh);
3819
3820     my $res = '';
3821     my $buf;
3822
3823     my @ready;
3824     while (scalar (@ready = $sel->can_read($timeout))) {
3825         my $count;
3826         if ($count = $fh->sysread($buf, 8192)) {
3827             if ($buf =~ /^(.*)\(qemu\) $/s) {
3828                 $res .= $1;
3829                 last;
3830             } else {
3831                 $res .= $buf;
3832             }
3833         } else {
3834             if (!defined($count)) {
3835                 die "$!\n";
3836             }
3837             last;
3838         }
3839     }
3840
3841     die "monitor read timeout\n" if !scalar(@ready);
3842
3843     return $res;
3844 }
3845
3846 # old code, only used to shutdown old VM after update
3847 sub vm_monitor_command {
3848     my ($vmid, $cmdstr, $nocheck) = @_;
3849
3850     my $res;
3851
3852     eval {
3853         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3854
3855         my $sname = "${var_run_tmpdir}/$vmid.mon";
3856
3857         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3858             die "unable to connect to VM $vmid socket - $!\n";
3859
3860         my $timeout = 3;
3861
3862         # hack: migrate sometime blocks the monitor (when migrate_downtime
3863         # is set)
3864         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3865             $timeout = 60*60; # 1 hour
3866         }
3867
3868         # read banner;
3869         my $data = __read_avail($sock, $timeout);
3870
3871         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3872             die "got unexpected qemu monitor banner\n";
3873         }
3874
3875         my $sel = new IO::Select;
3876         $sel->add($sock);
3877
3878         if (!scalar(my @ready = $sel->can_write($timeout))) {
3879             die "monitor write error - timeout";
3880         }
3881
3882         my $fullcmd = "$cmdstr\r";
3883
3884         # syslog('info', "VM $vmid monitor command: $cmdstr");
3885
3886         my $b;
3887         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3888             die "monitor write error - $!";
3889         }
3890
3891         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3892
3893         $timeout = 20;
3894
3895         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3896             $timeout = 60*60; # 1 hour
3897         } elsif ($cmdstr =~ m/^(eject|change)/) {
3898             $timeout = 60; # note: cdrom mount command is slow
3899         }
3900         if ($res = __read_avail($sock, $timeout)) {
3901
3902             my @lines = split("\r?\n", $res);
3903
3904             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3905
3906             $res = join("\n", @lines);
3907             $res .= "\n";
3908         }
3909     };
3910
3911     my $err = $@;
3912
3913     if ($err) {
3914         syslog("err", "VM $vmid monitor command failed - $err");
3915         die $err;
3916     }
3917
3918     return $res;
3919 }
3920
3921 sub qemu_block_resize {
3922     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3923
3924     my $running = check_running($vmid);
3925
3926     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3927
3928     return if !$running;
3929
3930     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3931
3932 }
3933
3934 sub qemu_volume_snapshot {
3935     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3936
3937     my $running = check_running($vmid);
3938
3939     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3940         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3941     } else {
3942         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3943     }
3944 }
3945
3946 sub qemu_volume_snapshot_delete {
3947     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3948
3949     my $running = check_running($vmid);
3950
3951     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3952
3953     return if !$running;
3954
3955     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3956 }
3957
3958 sub set_migration_caps {
3959     my ($vmid) = @_;
3960
3961     my $cap_ref = [];
3962
3963     my $enabled_cap = {
3964         "auto-converge" => 1,
3965         "xbzrle" => 1,
3966         "x-rdma-pin-all" => 0,
3967         "zero-blocks" => 0,
3968         "compress" => 0
3969     };
3970
3971     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3972
3973     for my $supported_capability (@$supported_capabilities) {
3974         push @$cap_ref, {
3975             capability => $supported_capability->{capability},
3976             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3977         };
3978     }
3979
3980     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3981 }
3982
3983 my $fast_plug_option = {
3984     'lock' => 1,
3985     'name' => 1,
3986     'onboot' => 1,
3987     'shares' => 1,
3988     'startup' => 1,
3989     'description' => 1,
3990 };
3991
3992 # hotplug changes in [PENDING]
3993 # $selection hash can be used to only apply specified options, for
3994 # example: { cores => 1 } (only apply changed 'cores')
3995 # $errors ref is used to return error messages
3996 sub vmconfig_hotplug_pending {
3997     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3998
3999     my $defaults = load_defaults();
4000
4001     # commit values which do not have any impact on running VM first
4002     # Note: those option cannot raise errors, we we do not care about
4003     # $selection and always apply them.
4004
4005     my $add_error = sub {
4006         my ($opt, $msg) = @_;
4007         $errors->{$opt} = "hotplug problem - $msg";
4008     };
4009
4010     my $changes = 0;
4011     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4012         if ($fast_plug_option->{$opt}) {
4013             $conf->{$opt} = $conf->{pending}->{$opt};
4014             delete $conf->{pending}->{$opt};
4015             $changes = 1;
4016         }
4017     }
4018
4019     if ($changes) {
4020         PVE::QemuConfig->write_config($vmid, $conf);
4021         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4022     }
4023
4024     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4025
4026     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4027     while (my ($opt, $force) = each %$pending_delete_hash) {
4028         next if $selection && !$selection->{$opt};
4029         eval {
4030             if ($opt eq 'hotplug') {
4031                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4032             } elsif ($opt eq 'tablet') {
4033                 die "skip\n" if !$hotplug_features->{usb};
4034                 if ($defaults->{tablet}) {
4035                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4036                 } else {
4037                     vm_deviceunplug($vmid, $conf, $opt);
4038                 }
4039             } elsif ($opt eq 'vcpus') {
4040                 die "skip\n" if !$hotplug_features->{cpu};
4041                 qemu_cpu_hotplug($vmid, $conf, undef);
4042             } elsif ($opt eq 'balloon') {
4043                 # enable balloon device is not hotpluggable
4044                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4045             } elsif ($fast_plug_option->{$opt}) {
4046                 # do nothing
4047             } elsif ($opt =~ m/^net(\d+)$/) {
4048                 die "skip\n" if !$hotplug_features->{network};
4049                 vm_deviceunplug($vmid, $conf, $opt);
4050             } elsif (is_valid_drivename($opt)) {
4051                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4052                 vm_deviceunplug($vmid, $conf, $opt);
4053                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4054             } elsif ($opt =~ m/^memory$/) {
4055                 die "skip\n" if !$hotplug_features->{memory};
4056                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4057             } elsif ($opt eq 'cpuunits') {
4058                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4059             } elsif ($opt eq 'cpulimit') {
4060                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4061             } else {
4062                 die "skip\n";
4063             }
4064         };
4065         if (my $err = $@) {
4066             &$add_error($opt, $err) if $err ne "skip\n";
4067         } else {
4068             # save new config if hotplug was successful
4069             delete $conf->{$opt};
4070             vmconfig_undelete_pending_option($conf, $opt);
4071             PVE::QemuConfig->write_config($vmid, $conf);
4072             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4073         }
4074     }
4075
4076     foreach my $opt (keys %{$conf->{pending}}) {
4077         next if $selection && !$selection->{$opt};
4078         my $value = $conf->{pending}->{$opt};
4079         eval {
4080             if ($opt eq 'hotplug') {
4081                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4082             } elsif ($opt eq 'tablet') {
4083                 die "skip\n" if !$hotplug_features->{usb};
4084                 if ($value == 1) {
4085                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4086                 } elsif ($value == 0) {
4087                     vm_deviceunplug($vmid, $conf, $opt);
4088                 }
4089             } elsif ($opt eq 'vcpus') {
4090                 die "skip\n" if !$hotplug_features->{cpu};
4091                 qemu_cpu_hotplug($vmid, $conf, $value);
4092             } elsif ($opt eq 'balloon') {
4093                 # enable/disable balloning device is not hotpluggable
4094                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4095                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4096                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4097
4098                 # allow manual ballooning if shares is set to zero
4099                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4100                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4101                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4102                 }
4103             } elsif ($opt =~ m/^net(\d+)$/) {
4104                 # some changes can be done without hotplug
4105                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4106                                     $vmid, $opt, $value);
4107             } elsif (is_valid_drivename($opt)) {
4108                 # some changes can be done without hotplug
4109                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4110                                      $vmid, $opt, $value, 1);
4111             } elsif ($opt =~ m/^memory$/) { #dimms
4112                 die "skip\n" if !$hotplug_features->{memory};
4113                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4114             } elsif ($opt eq 'cpuunits') {
4115                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4116             } elsif ($opt eq 'cpulimit') {
4117                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4118                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4119             } else {
4120                 die "skip\n";  # skip non-hot-pluggable options
4121             }
4122         };
4123         if (my $err = $@) {
4124             &$add_error($opt, $err) if $err ne "skip\n";
4125         } else {
4126             # save new config if hotplug was successful
4127             $conf->{$opt} = $value;
4128             delete $conf->{pending}->{$opt};
4129             PVE::QemuConfig->write_config($vmid, $conf);
4130             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4131         }
4132     }
4133 }
4134
4135 sub try_deallocate_drive {
4136     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4137
4138     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4139         my $volid = $drive->{file};
4140         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4141             my $sid = PVE::Storage::parse_volume_id($volid);
4142             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4143
4144             # check if the disk is really unused
4145             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4146                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4147             PVE::Storage::vdisk_free($storecfg, $volid);
4148             return 1;
4149         } else {
4150             # If vm is not owner of this disk remove from config
4151             return 1;
4152         }
4153     }
4154
4155     return undef;
4156 }
4157
4158 sub vmconfig_delete_or_detach_drive {
4159     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4160
4161     my $drive = parse_drive($opt, $conf->{$opt});
4162
4163     my $rpcenv = PVE::RPCEnvironment::get();
4164     my $authuser = $rpcenv->get_user();
4165
4166     if ($force) {
4167         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4168         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4169     } else {
4170         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4171     }
4172 }
4173
4174 sub vmconfig_apply_pending {
4175     my ($vmid, $conf, $storecfg) = @_;
4176
4177     # cold plug
4178
4179     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4180     while (my ($opt, $force) = each %$pending_delete_hash) {
4181         die "internal error" if $opt =~ m/^unused/;
4182         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4183         if (!defined($conf->{$opt})) {
4184             vmconfig_undelete_pending_option($conf, $opt);
4185             PVE::QemuConfig->write_config($vmid, $conf);
4186         } elsif (is_valid_drivename($opt)) {
4187             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4188             vmconfig_undelete_pending_option($conf, $opt);
4189             delete $conf->{$opt};
4190             PVE::QemuConfig->write_config($vmid, $conf);
4191         } else {
4192             vmconfig_undelete_pending_option($conf, $opt);
4193             delete $conf->{$opt};
4194             PVE::QemuConfig->write_config($vmid, $conf);
4195         }
4196     }
4197
4198     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4199
4200     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4201         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4202
4203         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4204             # skip if nothing changed
4205         } elsif (is_valid_drivename($opt)) {
4206             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4207                 if defined($conf->{$opt});
4208             $conf->{$opt} = $conf->{pending}->{$opt};
4209         } else {
4210             $conf->{$opt} = $conf->{pending}->{$opt};
4211         }
4212
4213         delete $conf->{pending}->{$opt};
4214         PVE::QemuConfig->write_config($vmid, $conf);
4215     }
4216 }
4217
4218 my $safe_num_ne = sub {
4219     my ($a, $b) = @_;
4220
4221     return 0 if !defined($a) && !defined($b);
4222     return 1 if !defined($a);
4223     return 1 if !defined($b);
4224
4225     return $a != $b;
4226 };
4227
4228 my $safe_string_ne = sub {
4229     my ($a, $b) = @_;
4230
4231     return 0 if !defined($a) && !defined($b);
4232     return 1 if !defined($a);
4233     return 1 if !defined($b);
4234
4235     return $a ne $b;
4236 };
4237
4238 sub vmconfig_update_net {
4239     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4240
4241     my $newnet = parse_net($value);
4242
4243     if ($conf->{$opt}) {
4244         my $oldnet = parse_net($conf->{$opt});
4245
4246         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4247             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4248             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4249             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4250
4251             # for non online change, we try to hot-unplug
4252             die "skip\n" if !$hotplug;
4253             vm_deviceunplug($vmid, $conf, $opt);
4254         } else {
4255
4256             die "internal error" if $opt !~ m/net(\d+)/;
4257             my $iface = "tap${vmid}i$1";
4258
4259             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4260                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4261                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4262                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4263                 PVE::Network::tap_unplug($iface);
4264                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4265             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4266                 # Rate can be applied on its own but any change above needs to
4267                 # include the rate in tap_plug since OVS resets everything.
4268                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4269             }
4270
4271             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4272                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4273             }
4274
4275             return 1;
4276         }
4277     }
4278
4279     if ($hotplug) {
4280         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4281     } else {
4282         die "skip\n";
4283     }
4284 }
4285
4286 sub vmconfig_update_disk {
4287     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4288
4289     # fixme: do we need force?
4290
4291     my $drive = parse_drive($opt, $value);
4292
4293     if ($conf->{$opt}) {
4294
4295         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4296
4297             my $media = $drive->{media} || 'disk';
4298             my $oldmedia = $old_drive->{media} || 'disk';
4299             die "unable to change media type\n" if $media ne $oldmedia;
4300
4301             if (!drive_is_cdrom($old_drive)) {
4302
4303                 if ($drive->{file} ne $old_drive->{file}) {
4304
4305                     die "skip\n" if !$hotplug;
4306
4307                     # unplug and register as unused
4308                     vm_deviceunplug($vmid, $conf, $opt);
4309                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4310
4311                 } else {
4312                     # update existing disk
4313
4314                     # skip non hotpluggable value
4315                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4316                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4317                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4318                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4319                         die "skip\n";
4320                     }
4321
4322                     # apply throttle
4323                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4324                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4325                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4326                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4327                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4328                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4329                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4330                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4331                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4332                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4333                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4334                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4335
4336                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4337                                                    ($drive->{mbps} || 0)*1024*1024,
4338                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4339                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4340                                                    $drive->{iops} || 0,
4341                                                    $drive->{iops_rd} || 0,
4342                                                    $drive->{iops_wr} || 0,
4343                                                    ($drive->{mbps_max} || 0)*1024*1024,
4344                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4345                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4346                                                    $drive->{iops_max} || 0,
4347                                                    $drive->{iops_rd_max} || 0,
4348                                                    $drive->{iops_wr_max} || 0);
4349
4350                     }
4351
4352                     return 1;
4353                 }
4354
4355             } else { # cdrom
4356
4357                 if ($drive->{file} eq 'none') {
4358                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4359                 } else {
4360                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4361                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4362                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4363                 }
4364
4365                 return 1;
4366             }
4367         }
4368     }
4369
4370     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4371     # hotplug new disks
4372     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4373     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4374 }
4375
4376 sub vm_start {
4377     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4378         $forcemachine, $spice_ticket) = @_;
4379
4380     PVE::QemuConfig->lock_config($vmid, sub {
4381         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4382
4383         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4384
4385         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4386
4387         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4388
4389         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4390             vmconfig_apply_pending($vmid, $conf, $storecfg);
4391             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4392         }
4393
4394         my $defaults = load_defaults();
4395
4396         # set environment variable useful inside network script
4397         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4398
4399         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4400
4401         my $migrate_port = 0;
4402         my $migrate_uri;
4403         if ($statefile) {
4404             if ($statefile eq 'tcp') {
4405                 my $localip = "localhost";
4406                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4407                 my $nodename = PVE::INotify::nodename();
4408                 if ($datacenterconf->{migration_unsecure}) {
4409                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4410                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4411                 }
4412                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4413                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4414                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4415                 push @$cmd, '-incoming', $migrate_uri;
4416                 push @$cmd, '-S';
4417             } else {
4418                 push @$cmd, '-loadstate', $statefile;
4419             }
4420         } elsif ($paused) {
4421             push @$cmd, '-S';
4422         }
4423
4424         # host pci devices
4425         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4426           my $d = parse_hostpci($conf->{"hostpci$i"});
4427           next if !$d;
4428           my $pcidevices = $d->{pciid};
4429           foreach my $pcidevice (@$pcidevices) {
4430                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4431
4432                 my $info = pci_device_info("0000:$pciid");
4433                 die "IOMMU not present\n" if !check_iommu_support();
4434                 die "no pci device info for device '$pciid'\n" if !$info;
4435                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4436                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4437           }
4438         }
4439
4440         PVE::Storage::activate_volumes($storecfg, $vollist);
4441
4442         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4443             my $cmd = [];
4444             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4445             eval  { run_command($cmd); };
4446         }
4447
4448         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4449                     umask => 0077); };
4450
4451         if (my $err = $@) {
4452             # deactivate volumes if start fails
4453             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4454             die "start failed: $err";
4455         }
4456
4457         print "migration listens on $migrate_uri\n" if $migrate_uri;
4458
4459         if ($statefile && $statefile ne 'tcp')  {
4460             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4461             warn $@ if $@;
4462         }
4463
4464         if ($migratedfrom) {
4465
4466             eval {
4467                 set_migration_caps($vmid);
4468             };
4469             warn $@ if $@;
4470
4471             if ($spice_port) {
4472                 print "spice listens on port $spice_port\n";
4473                 if ($spice_ticket) {
4474                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4475                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4476                 }
4477             }
4478
4479         } else {
4480
4481             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4482                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4483                     if $conf->{balloon};
4484             }
4485
4486             foreach my $opt (keys %$conf) {
4487                 next if $opt !~  m/^net\d+$/;
4488                 my $nicconf = parse_net($conf->{$opt});
4489                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4490             }
4491         }
4492
4493         vm_mon_cmd_nocheck($vmid, 'qom-set',
4494                     path => "machine/peripheral/balloon0",
4495                     property => "guest-stats-polling-interval",
4496                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4497
4498     });
4499 }
4500
4501 sub vm_mon_cmd {
4502     my ($vmid, $execute, %params) = @_;
4503
4504     my $cmd = { execute => $execute, arguments => \%params };
4505     vm_qmp_command($vmid, $cmd);
4506 }
4507
4508 sub vm_mon_cmd_nocheck {
4509     my ($vmid, $execute, %params) = @_;
4510
4511     my $cmd = { execute => $execute, arguments => \%params };
4512     vm_qmp_command($vmid, $cmd, 1);
4513 }
4514
4515 sub vm_qmp_command {
4516     my ($vmid, $cmd, $nocheck) = @_;
4517
4518     my $res;
4519
4520     my $timeout;
4521     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4522         $timeout = $cmd->{arguments}->{timeout};
4523         delete $cmd->{arguments}->{timeout};
4524     }
4525
4526     eval {
4527         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4528         my $sname = qmp_socket($vmid);
4529         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4530             my $qmpclient = PVE::QMPClient->new();
4531
4532             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4533         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4534             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4535                 if scalar(%{$cmd->{arguments}});
4536             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4537         } else {
4538             die "unable to open monitor socket\n";
4539         }
4540     };
4541     if (my $err = $@) {
4542         syslog("err", "VM $vmid qmp command failed - $err");
4543         die $err;
4544     }
4545
4546     return $res;
4547 }
4548
4549 sub vm_human_monitor_command {
4550     my ($vmid, $cmdline) = @_;
4551
4552     my $res;
4553
4554     my $cmd = {
4555         execute => 'human-monitor-command',
4556         arguments => { 'command-line' => $cmdline},
4557     };
4558
4559     return vm_qmp_command($vmid, $cmd);
4560 }
4561
4562 sub vm_commandline {
4563     my ($storecfg, $vmid) = @_;
4564
4565     my $conf = PVE::QemuConfig->load_config($vmid);
4566
4567     my $defaults = load_defaults();
4568
4569     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4570
4571     return join(' ', @$cmd);
4572 }
4573
4574 sub vm_reset {
4575     my ($vmid, $skiplock) = @_;
4576
4577     PVE::QemuConfig->lock_config($vmid, sub {
4578
4579         my $conf = PVE::QemuConfig->load_config($vmid);
4580
4581         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4582
4583         vm_mon_cmd($vmid, "system_reset");
4584     });
4585 }
4586
4587 sub get_vm_volumes {
4588     my ($conf) = @_;
4589
4590     my $vollist = [];
4591     foreach_volid($conf, sub {
4592         my ($volid, $is_cdrom) = @_;
4593
4594         return if $volid =~ m|^/|;
4595
4596         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4597         return if !$sid;
4598
4599         push @$vollist, $volid;
4600     });
4601
4602     return $vollist;
4603 }
4604
4605 sub vm_stop_cleanup {
4606     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4607
4608     eval {
4609
4610         if (!$keepActive) {
4611             my $vollist = get_vm_volumes($conf);
4612             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4613         }
4614
4615         foreach my $ext (qw(mon qmp pid vnc qga)) {
4616             unlink "/var/run/qemu-server/${vmid}.$ext";
4617         }
4618
4619         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4620     };
4621     warn $@ if $@; # avoid errors - just warn
4622 }
4623
4624 # Note: use $nockeck to skip tests if VM configuration file exists.
4625 # We need that when migration VMs to other nodes (files already moved)
4626 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4627 sub vm_stop {
4628     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4629
4630     $force = 1 if !defined($force) && !$shutdown;
4631
4632     if ($migratedfrom){
4633         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4634         kill 15, $pid if $pid;
4635         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4636         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4637         return;
4638     }
4639
4640     PVE::QemuConfig->lock_config($vmid, sub {
4641
4642         my $pid = check_running($vmid, $nocheck);
4643         return if !$pid;
4644
4645         my $conf;
4646         if (!$nocheck) {
4647             $conf = PVE::QemuConfig->load_config($vmid);
4648             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4649             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4650                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4651                 $timeout = $opts->{down} if $opts->{down};
4652             }
4653         }
4654
4655         $timeout = 60 if !defined($timeout);
4656
4657         eval {
4658             if ($shutdown) {
4659                 if (defined($conf) && $conf->{agent}) {
4660                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4661                 } else {
4662                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4663                 }
4664             } else {
4665                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4666             }
4667         };
4668         my $err = $@;
4669
4670         if (!$err) {
4671             my $count = 0;
4672             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4673                 $count++;
4674                 sleep 1;
4675             }
4676
4677             if ($count >= $timeout) {
4678                 if ($force) {
4679                     warn "VM still running - terminating now with SIGTERM\n";
4680                     kill 15, $pid;
4681                 } else {
4682                     die "VM quit/powerdown failed - got timeout\n";
4683                 }
4684             } else {
4685                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4686                 return;
4687             }
4688         } else {
4689             if ($force) {
4690                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4691                 kill 15, $pid;
4692             } else {
4693                 die "VM quit/powerdown failed\n";
4694             }
4695         }
4696
4697         # wait again
4698         $timeout = 10;
4699
4700         my $count = 0;
4701         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4702             $count++;
4703             sleep 1;
4704         }
4705
4706         if ($count >= $timeout) {
4707             warn "VM still running - terminating now with SIGKILL\n";
4708             kill 9, $pid;
4709             sleep 1;
4710         }
4711
4712         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4713    });
4714 }
4715
4716 sub vm_suspend {
4717     my ($vmid, $skiplock) = @_;
4718
4719     PVE::QemuConfig->lock_config($vmid, sub {
4720
4721         my $conf = PVE::QemuConfig->load_config($vmid);
4722
4723         PVE::QemuConfig->check_lock($conf)
4724             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4725
4726         vm_mon_cmd($vmid, "stop");
4727     });
4728 }
4729
4730 sub vm_resume {
4731     my ($vmid, $skiplock, $nocheck) = @_;
4732
4733     PVE::QemuConfig->lock_config($vmid, sub {
4734
4735         if (!$nocheck) {
4736
4737             my $conf = PVE::QemuConfig->load_config($vmid);
4738
4739             PVE::QemuConfig->check_lock($conf)
4740                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4741
4742             vm_mon_cmd($vmid, "cont");
4743
4744         } else {
4745             vm_mon_cmd_nocheck($vmid, "cont");
4746         }
4747     });
4748 }
4749
4750 sub vm_sendkey {
4751     my ($vmid, $skiplock, $key) = @_;
4752
4753     PVE::QemuConfig->lock_config($vmid, sub {
4754
4755         my $conf = PVE::QemuConfig->load_config($vmid);
4756
4757         # there is no qmp command, so we use the human monitor command
4758         vm_human_monitor_command($vmid, "sendkey $key");
4759     });
4760 }
4761
4762 sub vm_destroy {
4763     my ($storecfg, $vmid, $skiplock) = @_;
4764
4765     PVE::QemuConfig->lock_config($vmid, sub {
4766
4767         my $conf = PVE::QemuConfig->load_config($vmid);
4768
4769         if (!check_running($vmid)) {
4770             destroy_vm($storecfg, $vmid, undef, $skiplock);
4771         } else {
4772             die "VM $vmid is running - destroy failed\n";
4773         }
4774     });
4775 }
4776
4777 # pci helpers
4778
4779 sub file_write {
4780     my ($filename, $buf) = @_;
4781
4782     my $fh = IO::File->new($filename, "w");
4783     return undef if !$fh;
4784
4785     my $res = print $fh $buf;
4786
4787     $fh->close();
4788
4789     return $res;
4790 }
4791
4792 sub pci_device_info {
4793     my ($name) = @_;
4794
4795     my $res;
4796
4797     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4798     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4799
4800     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4801     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4802
4803     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4804     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4805
4806     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4807     return undef if !defined($product) || $product !~ s/^0x//;
4808
4809     $res = {
4810         name => $name,
4811         vendor => $vendor,
4812         product => $product,
4813         domain => $domain,
4814         bus => $bus,
4815         slot => $slot,
4816         func => $func,
4817         irq => $irq,
4818         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4819     };
4820
4821     return $res;
4822 }
4823
4824 sub pci_dev_reset {
4825     my ($dev) = @_;
4826
4827     my $name = $dev->{name};
4828
4829     my $fn = "$pcisysfs/devices/$name/reset";
4830
4831     return file_write($fn, "1");
4832 }
4833
4834 sub pci_dev_bind_to_vfio {
4835     my ($dev) = @_;
4836
4837     my $name = $dev->{name};
4838
4839     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4840
4841     if (!-d $vfio_basedir) {
4842         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4843     }
4844     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4845
4846     my $testdir = "$vfio_basedir/$name";
4847     return 1 if -d $testdir;
4848
4849     my $data = "$dev->{vendor} $dev->{product}";
4850     return undef if !file_write("$vfio_basedir/new_id", $data);
4851
4852     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4853     if (!file_write($fn, $name)) {
4854         return undef if -f $fn;
4855     }
4856
4857     $fn = "$vfio_basedir/bind";
4858     if (! -d $testdir) {
4859         return undef if !file_write($fn, $name);
4860     }
4861
4862     return -d $testdir;
4863 }
4864
4865 sub pci_dev_group_bind_to_vfio {
4866     my ($pciid) = @_;
4867
4868     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4869
4870     if (!-d $vfio_basedir) {
4871         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4872     }
4873     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4874
4875     # get IOMMU group devices
4876     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4877       my @devs = grep /^0000:/, readdir($D);
4878     closedir($D);
4879
4880     foreach my $pciid (@devs) {
4881         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4882
4883         # pci bridges, switches or root ports are not supported
4884         # they have a pci_bus subdirectory so skip them
4885         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4886
4887         my $info = pci_device_info($1);
4888         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4889     }
4890
4891     return 1;
4892 }
4893
4894 sub print_pci_addr {
4895     my ($id, $bridges) = @_;
4896
4897     my $res = '';
4898     my $devices = {
4899         piix3 => { bus => 0, addr => 1 },
4900         #addr2 : first videocard
4901         balloon0 => { bus => 0, addr => 3 },
4902         watchdog => { bus => 0, addr => 4 },
4903         scsihw0 => { bus => 0, addr => 5 },
4904         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4905         scsihw1 => { bus => 0, addr => 6 },
4906         ahci0 => { bus => 0, addr => 7 },
4907         qga0 => { bus => 0, addr => 8 },
4908         spice => { bus => 0, addr => 9 },
4909         virtio0 => { bus => 0, addr => 10 },
4910         virtio1 => { bus => 0, addr => 11 },
4911         virtio2 => { bus => 0, addr => 12 },
4912         virtio3 => { bus => 0, addr => 13 },
4913         virtio4 => { bus => 0, addr => 14 },
4914         virtio5 => { bus => 0, addr => 15 },
4915         hostpci0 => { bus => 0, addr => 16 },
4916         hostpci1 => { bus => 0, addr => 17 },
4917         net0 => { bus => 0, addr => 18 },
4918         net1 => { bus => 0, addr => 19 },
4919         net2 => { bus => 0, addr => 20 },
4920         net3 => { bus => 0, addr => 21 },
4921         net4 => { bus => 0, addr => 22 },
4922         net5 => { bus => 0, addr => 23 },
4923         vga1 => { bus => 0, addr => 24 },
4924         vga2 => { bus => 0, addr => 25 },
4925         vga3 => { bus => 0, addr => 26 },
4926         hostpci2 => { bus => 0, addr => 27 },
4927         hostpci3 => { bus => 0, addr => 28 },
4928         #addr29 : usb-host (pve-usb.cfg)
4929         'pci.1' => { bus => 0, addr => 30 },
4930         'pci.2' => { bus => 0, addr => 31 },
4931         'net6' => { bus => 1, addr => 1 },
4932         'net7' => { bus => 1, addr => 2 },
4933         'net8' => { bus => 1, addr => 3 },
4934         'net9' => { bus => 1, addr => 4 },
4935         'net10' => { bus => 1, addr => 5 },
4936         'net11' => { bus => 1, addr => 6 },
4937         'net12' => { bus => 1, addr => 7 },
4938         'net13' => { bus => 1, addr => 8 },
4939         'net14' => { bus => 1, addr => 9 },
4940         'net15' => { bus => 1, addr => 10 },
4941         'net16' => { bus => 1, addr => 11 },
4942         'net17' => { bus => 1, addr => 12 },
4943         'net18' => { bus => 1, addr => 13 },
4944         'net19' => { bus => 1, addr => 14 },
4945         'net20' => { bus => 1, addr => 15 },
4946         'net21' => { bus => 1, addr => 16 },
4947         'net22' => { bus => 1, addr => 17 },
4948         'net23' => { bus => 1, addr => 18 },
4949         'net24' => { bus => 1, addr => 19 },
4950         'net25' => { bus => 1, addr => 20 },
4951         'net26' => { bus => 1, addr => 21 },
4952         'net27' => { bus => 1, addr => 22 },
4953         'net28' => { bus => 1, addr => 23 },
4954         'net29' => { bus => 1, addr => 24 },
4955         'net30' => { bus => 1, addr => 25 },
4956         'net31' => { bus => 1, addr => 26 },
4957         'xhci' => { bus => 1, addr => 27 },
4958         'virtio6' => { bus => 2, addr => 1 },
4959         'virtio7' => { bus => 2, addr => 2 },
4960         'virtio8' => { bus => 2, addr => 3 },
4961         'virtio9' => { bus => 2, addr => 4 },
4962         'virtio10' => { bus => 2, addr => 5 },
4963         'virtio11' => { bus => 2, addr => 6 },
4964         'virtio12' => { bus => 2, addr => 7 },
4965         'virtio13' => { bus => 2, addr => 8 },
4966         'virtio14' => { bus => 2, addr => 9 },
4967         'virtio15' => { bus => 2, addr => 10 },
4968         'virtioscsi0' => { bus => 3, addr => 1 },
4969         'virtioscsi1' => { bus => 3, addr => 2 },
4970         'virtioscsi2' => { bus => 3, addr => 3 },
4971         'virtioscsi3' => { bus => 3, addr => 4 },
4972         'virtioscsi4' => { bus => 3, addr => 5 },
4973         'virtioscsi5' => { bus => 3, addr => 6 },
4974         'virtioscsi6' => { bus => 3, addr => 7 },
4975         'virtioscsi7' => { bus => 3, addr => 8 },
4976         'virtioscsi8' => { bus => 3, addr => 9 },
4977         'virtioscsi9' => { bus => 3, addr => 10 },
4978         'virtioscsi10' => { bus => 3, addr => 11 },
4979         'virtioscsi11' => { bus => 3, addr => 12 },
4980         'virtioscsi12' => { bus => 3, addr => 13 },
4981         'virtioscsi13' => { bus => 3, addr => 14 },
4982         'virtioscsi14' => { bus => 3, addr => 15 },
4983         'virtioscsi15' => { bus => 3, addr => 16 },
4984         'virtioscsi16' => { bus => 3, addr => 17 },
4985         'virtioscsi17' => { bus => 3, addr => 18 },
4986         'virtioscsi18' => { bus => 3, addr => 19 },
4987         'virtioscsi19' => { bus => 3, addr => 20 },
4988         'virtioscsi20' => { bus => 3, addr => 21 },
4989         'virtioscsi21' => { bus => 3, addr => 22 },
4990         'virtioscsi22' => { bus => 3, addr => 23 },
4991         'virtioscsi23' => { bus => 3, addr => 24 },
4992         'virtioscsi24' => { bus => 3, addr => 25 },
4993         'virtioscsi25' => { bus => 3, addr => 26 },
4994         'virtioscsi26' => { bus => 3, addr => 27 },
4995         'virtioscsi27' => { bus => 3, addr => 28 },
4996         'virtioscsi28' => { bus => 3, addr => 29 },
4997         'virtioscsi29' => { bus => 3, addr => 30 },
4998         'virtioscsi30' => { bus => 3, addr => 31 },
4999
5000     };
5001
5002     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5003            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5004            my $bus = $devices->{$id}->{bus};
5005            $res = ",bus=pci.$bus,addr=$addr";
5006            $bridges->{$bus} = 1 if $bridges;
5007     }
5008     return $res;
5009
5010 }
5011
5012 sub print_pcie_addr {
5013     my ($id) = @_;
5014
5015     my $res = '';
5016     my $devices = {
5017         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5018         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5019         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5020         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5021     };
5022
5023     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5024            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5025            my $bus = $devices->{$id}->{bus};
5026            $res = ",bus=$bus,addr=$addr";
5027     }
5028     return $res;
5029
5030 }
5031
5032 # vzdump restore implementaion
5033
5034 sub tar_archive_read_firstfile {
5035     my $archive = shift;
5036
5037     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5038
5039     # try to detect archive type first
5040     my $pid = open (TMP, "tar tf '$archive'|") ||
5041         die "unable to open file '$archive'\n";
5042     my $firstfile = <TMP>;
5043     kill 15, $pid;
5044     close TMP;
5045
5046     die "ERROR: archive contaions no data\n" if !$firstfile;
5047     chomp $firstfile;
5048
5049     return $firstfile;
5050 }
5051
5052 sub tar_restore_cleanup {
5053     my ($storecfg, $statfile) = @_;
5054
5055     print STDERR "starting cleanup\n";
5056
5057     if (my $fd = IO::File->new($statfile, "r")) {
5058         while (defined(my $line = <$fd>)) {
5059             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5060                 my $volid = $2;
5061                 eval {
5062                     if ($volid =~ m|^/|) {
5063                         unlink $volid || die 'unlink failed\n';
5064                     } else {
5065                         PVE::Storage::vdisk_free($storecfg, $volid);
5066                     }
5067                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5068                 };
5069                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5070             } else {
5071                 print STDERR "unable to parse line in statfile - $line";
5072             }
5073         }
5074         $fd->close();
5075     }
5076 }
5077
5078 sub restore_archive {
5079     my ($archive, $vmid, $user, $opts) = @_;
5080
5081     my $format = $opts->{format};
5082     my $comp;
5083
5084     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5085         $format = 'tar' if !$format;
5086         $comp = 'gzip';
5087     } elsif ($archive =~ m/\.tar$/) {
5088         $format = 'tar' if !$format;
5089     } elsif ($archive =~ m/.tar.lzo$/) {
5090         $format = 'tar' if !$format;
5091         $comp = 'lzop';
5092     } elsif ($archive =~ m/\.vma$/) {
5093         $format = 'vma' if !$format;
5094     } elsif ($archive =~ m/\.vma\.gz$/) {
5095         $format = 'vma' if !$format;
5096         $comp = 'gzip';
5097     } elsif ($archive =~ m/\.vma\.lzo$/) {
5098         $format = 'vma' if !$format;
5099         $comp = 'lzop';
5100     } else {
5101         $format = 'vma' if !$format; # default
5102     }
5103
5104     # try to detect archive format
5105     if ($format eq 'tar') {
5106         return restore_tar_archive($archive, $vmid, $user, $opts);
5107     } else {
5108         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5109     }
5110 }
5111
5112 sub restore_update_config_line {
5113     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5114
5115     return if $line =~ m/^\#qmdump\#/;
5116     return if $line =~ m/^\#vzdump\#/;
5117     return if $line =~ m/^lock:/;
5118     return if $line =~ m/^unused\d+:/;
5119     return if $line =~ m/^parent:/;
5120     return if $line =~ m/^template:/; # restored VM is never a template
5121
5122     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5123         # try to convert old 1.X settings
5124         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5125         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5126             my ($model, $macaddr) = split(/\=/, $devconfig);
5127             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5128             my $net = {
5129                 model => $model,
5130                 bridge => "vmbr$ind",
5131                 macaddr => $macaddr,
5132             };
5133             my $netstr = print_net($net);
5134
5135             print $outfd "net$cookie->{netcount}: $netstr\n";
5136             $cookie->{netcount}++;
5137         }
5138     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5139         my ($id, $netstr) = ($1, $2);
5140         my $net = parse_net($netstr);
5141         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5142         $netstr = print_net($net);
5143         print $outfd "$id: $netstr\n";
5144     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5145         my $virtdev = $1;
5146         my $value = $3;
5147         my $di = parse_drive($virtdev, $value);
5148         if (defined($di->{backup}) && !$di->{backup}) {
5149             print $outfd "#$line";
5150         } elsif ($map->{$virtdev}) {
5151             delete $di->{format}; # format can change on restore
5152             $di->{file} = $map->{$virtdev};
5153             $value = print_drive($vmid, $di);
5154             print $outfd "$virtdev: $value\n";
5155         } else {
5156             print $outfd $line;
5157         }
5158     } else {
5159         print $outfd $line;
5160     }
5161 }
5162
5163 sub scan_volids {
5164     my ($cfg, $vmid) = @_;
5165
5166     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5167
5168     my $volid_hash = {};
5169     foreach my $storeid (keys %$info) {
5170         foreach my $item (@{$info->{$storeid}}) {
5171             next if !($item->{volid} && $item->{size});
5172             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5173             $volid_hash->{$item->{volid}} = $item;
5174         }
5175     }
5176
5177     return $volid_hash;
5178 }
5179
5180 sub is_volume_in_use {
5181     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5182
5183     my $path = PVE::Storage::path($storecfg, $volid);
5184
5185     my $scan_config = sub {
5186         my ($cref, $snapname) = @_;
5187
5188         foreach my $key (keys %$cref) {
5189             my $value = $cref->{$key};
5190             if (is_valid_drivename($key)) {
5191                 next if $skip_drive && $key eq $skip_drive;
5192                 my $drive = parse_drive($key, $value);
5193                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5194                 return 1 if $volid eq $drive->{file};
5195                 if ($drive->{file} =~ m!^/!) {
5196                     return 1 if $drive->{file} eq $path;
5197                 } else {
5198                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5199                     next if !$storeid;
5200                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5201                     next if !$scfg;
5202                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5203                 }
5204             }
5205         }
5206
5207         return 0;
5208     };
5209
5210     return 1 if &$scan_config($conf);
5211
5212     undef $skip_drive;
5213
5214     foreach my $snapname (keys %{$conf->{snapshots}}) {
5215         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5216     }
5217
5218     return 0;
5219 }
5220
5221 sub update_disksize {
5222     my ($vmid, $conf, $volid_hash) = @_;
5223
5224     my $changes;
5225
5226     my $used = {};
5227
5228     # Note: it is allowed to define multiple storages with same path (alias), so
5229     # we need to check both 'volid' and real 'path' (two different volid can point
5230     # to the same path).
5231
5232     my $usedpath = {};
5233
5234     # update size info
5235     foreach my $opt (keys %$conf) {
5236         if (is_valid_drivename($opt)) {
5237             my $drive = parse_drive($opt, $conf->{$opt});
5238             my $volid = $drive->{file};
5239             next if !$volid;
5240
5241             $used->{$volid} = 1;
5242             if ($volid_hash->{$volid} &&
5243                 (my $path = $volid_hash->{$volid}->{path})) {
5244                 $usedpath->{$path} = 1;
5245             }
5246
5247             next if drive_is_cdrom($drive);
5248             next if !$volid_hash->{$volid};
5249
5250             $drive->{size} = $volid_hash->{$volid}->{size};
5251             my $new = print_drive($vmid, $drive);
5252             if ($new ne $conf->{$opt}) {
5253                 $changes = 1;
5254                 $conf->{$opt} = $new;
5255             }
5256         }
5257     }
5258
5259     # remove 'unusedX' entry if volume is used
5260     foreach my $opt (keys %$conf) {
5261         next if $opt !~ m/^unused\d+$/;
5262         my $volid = $conf->{$opt};
5263         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5264         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5265             $changes = 1;
5266             delete $conf->{$opt};
5267         }
5268     }
5269
5270     foreach my $volid (sort keys %$volid_hash) {
5271         next if $volid =~ m/vm-$vmid-state-/;
5272         next if $used->{$volid};
5273         my $path = $volid_hash->{$volid}->{path};
5274         next if !$path; # just to be sure
5275         next if $usedpath->{$path};
5276         $changes = 1;
5277         PVE::QemuConfig->add_unused_volume($conf, $volid);
5278         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5279     }
5280
5281     return $changes;
5282 }
5283
5284 sub rescan {
5285     my ($vmid, $nolock) = @_;
5286
5287     my $cfg = PVE::Storage::config();
5288
5289     my $volid_hash = scan_volids($cfg, $vmid);
5290
5291     my $updatefn =  sub {
5292         my ($vmid) = @_;
5293
5294         my $conf = PVE::QemuConfig->load_config($vmid);
5295
5296         PVE::QemuConfig->check_lock($conf);
5297
5298         my $vm_volids = {};
5299         foreach my $volid (keys %$volid_hash) {
5300             my $info = $volid_hash->{$volid};
5301             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5302         }
5303
5304         my $changes = update_disksize($vmid, $conf, $vm_volids);
5305
5306         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5307     };
5308
5309     if (defined($vmid)) {
5310         if ($nolock) {
5311             &$updatefn($vmid);
5312         } else {
5313             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5314         }
5315     } else {
5316         my $vmlist = config_list();
5317         foreach my $vmid (keys %$vmlist) {
5318             if ($nolock) {
5319                 &$updatefn($vmid);
5320             } else {
5321                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5322             }
5323         }
5324     }
5325 }
5326
5327 sub restore_vma_archive {
5328     my ($archive, $vmid, $user, $opts, $comp) = @_;
5329
5330     my $input = $archive eq '-' ? "<&STDIN" : undef;
5331     my $readfrom = $archive;
5332
5333     my $uncomp = '';
5334     if ($comp) {
5335         $readfrom = '-';
5336         my $qarchive = PVE::Tools::shellquote($archive);
5337         if ($comp eq 'gzip') {
5338             $uncomp = "zcat $qarchive|";
5339         } elsif ($comp eq 'lzop') {
5340             $uncomp = "lzop -d -c $qarchive|";
5341         } else {
5342             die "unknown compression method '$comp'\n";
5343         }
5344
5345     }
5346
5347     my $tmpdir = "/var/tmp/vzdumptmp$$";
5348     rmtree $tmpdir;
5349
5350     # disable interrupts (always do cleanups)
5351     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5352         warn "got interrupt - ignored\n";
5353     };
5354
5355     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5356     POSIX::mkfifo($mapfifo, 0600);
5357     my $fifofh;
5358
5359     my $openfifo = sub {
5360         open($fifofh, '>', $mapfifo) || die $!;
5361     };
5362
5363     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5364
5365     my $oldtimeout;
5366     my $timeout = 5;
5367
5368     my $devinfo = {};
5369
5370     my $rpcenv = PVE::RPCEnvironment::get();
5371
5372     my $conffile = PVE::QemuConfig->config_file($vmid);
5373     my $tmpfn = "$conffile.$$.tmp";
5374
5375     # Note: $oldconf is undef if VM does not exists
5376     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5377     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5378
5379     my $print_devmap = sub {
5380         my $virtdev_hash = {};
5381
5382         my $cfgfn = "$tmpdir/qemu-server.conf";
5383
5384         # we can read the config - that is already extracted
5385         my $fh = IO::File->new($cfgfn, "r") ||
5386             "unable to read qemu-server.conf - $!\n";
5387
5388         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5389         if (-f $fwcfgfn) {
5390             my $pve_firewall_dir = '/etc/pve/firewall';
5391             mkdir $pve_firewall_dir; # make sure the dir exists
5392             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5393         }
5394
5395         while (defined(my $line = <$fh>)) {
5396             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5397                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5398                 die "archive does not contain data for drive '$virtdev'\n"
5399                     if !$devinfo->{$devname};
5400                 if (defined($opts->{storage})) {
5401                     $storeid = $opts->{storage} || 'local';
5402                 } elsif (!$storeid) {
5403                     $storeid = 'local';
5404                 }
5405                 $format = 'raw' if !$format;
5406                 $devinfo->{$devname}->{devname} = $devname;
5407                 $devinfo->{$devname}->{virtdev} = $virtdev;
5408                 $devinfo->{$devname}->{format} = $format;
5409                 $devinfo->{$devname}->{storeid} = $storeid;
5410
5411                 # check permission on storage
5412                 my $pool = $opts->{pool}; # todo: do we need that?
5413                 if ($user ne 'root@pam') {
5414                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5415                 }
5416
5417                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5418             }
5419         }
5420
5421         foreach my $devname (keys %$devinfo) {
5422             die "found no device mapping information for device '$devname'\n"
5423                 if !$devinfo->{$devname}->{virtdev};
5424         }
5425
5426         my $cfg = PVE::Storage::config();
5427
5428         # create empty/temp config
5429         if ($oldconf) {
5430             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5431             foreach_drive($oldconf, sub {
5432                 my ($ds, $drive) = @_;
5433
5434                 return if drive_is_cdrom($drive);
5435
5436                 my $volid = $drive->{file};
5437
5438                 return if !$volid || $volid =~ m|^/|;
5439
5440                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5441                 return if !$path || !$owner || ($owner != $vmid);
5442
5443                 # Note: only delete disk we want to restore
5444                 # other volumes will become unused
5445                 if ($virtdev_hash->{$ds}) {
5446                     PVE::Storage::vdisk_free($cfg, $volid);
5447                 }
5448             });
5449
5450             # delete vmstate files
5451             # since after the restore we have no snapshots anymore
5452             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5453                 my $snap = $oldconf->{snapshots}->{$snapname};
5454                 if ($snap->{vmstate}) {
5455                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5456                     if (my $err = $@) {
5457                         warn $err;
5458                     }
5459                 }
5460             }
5461         }
5462
5463         my $map = {};
5464         foreach my $virtdev (sort keys %$virtdev_hash) {
5465             my $d = $virtdev_hash->{$virtdev};
5466             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5467             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5468
5469             # test if requested format is supported
5470             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5471             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5472             $d->{format} = $defFormat if !$supported;
5473
5474             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5475                                                   $d->{format}, undef, $alloc_size);
5476             print STDERR "new volume ID is '$volid'\n";
5477             $d->{volid} = $volid;
5478             my $path = PVE::Storage::path($cfg, $volid);
5479
5480             PVE::Storage::activate_volumes($cfg,[$volid]);
5481
5482             my $write_zeros = 1;
5483             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5484                 $write_zeros = 0;
5485             }
5486
5487             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5488
5489             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5490             $map->{$virtdev} = $volid;
5491         }
5492
5493         $fh->seek(0, 0) || die "seek failed - $!\n";
5494
5495         my $outfd = new IO::File ($tmpfn, "w") ||
5496             die "unable to write config for VM $vmid\n";
5497
5498         my $cookie = { netcount => 0 };
5499         while (defined(my $line = <$fh>)) {
5500             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5501         }
5502
5503         $fh->close();
5504         $outfd->close();
5505     };
5506
5507     eval {
5508         # enable interrupts
5509         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5510             die "interrupted by signal\n";
5511         };
5512         local $SIG{ALRM} = sub { die "got timeout\n"; };
5513
5514         $oldtimeout = alarm($timeout);
5515
5516         my $parser = sub {
5517             my $line = shift;
5518
5519             print "$line\n";
5520
5521             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5522                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5523                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5524             } elsif ($line =~ m/^CTIME: /) {
5525                 # we correctly received the vma config, so we can disable
5526                 # the timeout now for disk allocation (set to 10 minutes, so
5527                 # that we always timeout if something goes wrong)
5528                 alarm(600);
5529                 &$print_devmap();
5530                 print $fifofh "done\n";
5531                 my $tmp = $oldtimeout || 0;
5532                 $oldtimeout = undef;
5533                 alarm($tmp);
5534                 close($fifofh);
5535             }
5536         };
5537
5538         print "restore vma archive: $cmd\n";
5539         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5540     };
5541     my $err = $@;
5542
5543     alarm($oldtimeout) if $oldtimeout;
5544
5545     my $vollist = [];
5546     foreach my $devname (keys %$devinfo) {
5547         my $volid = $devinfo->{$devname}->{volid};
5548         push @$vollist, $volid if $volid;
5549     }
5550
5551     my $cfg = PVE::Storage::config();
5552     PVE::Storage::deactivate_volumes($cfg, $vollist);
5553
5554     unlink $mapfifo;
5555
5556     if ($err) {
5557         rmtree $tmpdir;
5558         unlink $tmpfn;
5559
5560         foreach my $devname (keys %$devinfo) {
5561             my $volid = $devinfo->{$devname}->{volid};
5562             next if !$volid;
5563             eval {
5564                 if ($volid =~ m|^/|) {
5565                     unlink $volid || die 'unlink failed\n';
5566                 } else {
5567                     PVE::Storage::vdisk_free($cfg, $volid);
5568                 }
5569                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5570             };
5571             print STDERR "unable to cleanup '$volid' - $@" if $@;
5572         }
5573         die $err;
5574     }
5575
5576     rmtree $tmpdir;
5577
5578     rename($tmpfn, $conffile) ||
5579         die "unable to commit configuration file '$conffile'\n";
5580
5581     PVE::Cluster::cfs_update(); # make sure we read new file
5582
5583     eval { rescan($vmid, 1); };
5584     warn $@ if $@;
5585 }
5586
5587 sub restore_tar_archive {
5588     my ($archive, $vmid, $user, $opts) = @_;
5589
5590     if ($archive ne '-') {
5591         my $firstfile = tar_archive_read_firstfile($archive);
5592         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5593             if $firstfile ne 'qemu-server.conf';
5594     }
5595
5596     my $storecfg = PVE::Storage::config();
5597
5598     # destroy existing data - keep empty config
5599     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5600     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5601
5602     my $tocmd = "/usr/lib/qemu-server/qmextract";
5603
5604     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5605     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5606     $tocmd .= ' --prealloc' if $opts->{prealloc};
5607     $tocmd .= ' --info' if $opts->{info};
5608
5609     # tar option "xf" does not autodetect compression when read from STDIN,
5610     # so we pipe to zcat
5611     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5612         PVE::Tools::shellquote("--to-command=$tocmd");
5613
5614     my $tmpdir = "/var/tmp/vzdumptmp$$";
5615     mkpath $tmpdir;
5616
5617     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5618     local $ENV{VZDUMP_VMID} = $vmid;
5619     local $ENV{VZDUMP_USER} = $user;
5620
5621     my $conffile = PVE::QemuConfig->config_file($vmid);
5622     my $tmpfn = "$conffile.$$.tmp";
5623
5624     # disable interrupts (always do cleanups)
5625     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5626         print STDERR "got interrupt - ignored\n";
5627     };
5628
5629     eval {
5630         # enable interrupts
5631         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5632             die "interrupted by signal\n";
5633         };
5634
5635         if ($archive eq '-') {
5636             print "extracting archive from STDIN\n";
5637             run_command($cmd, input => "<&STDIN");
5638         } else {
5639             print "extracting archive '$archive'\n";
5640             run_command($cmd);
5641         }
5642
5643         return if $opts->{info};
5644
5645         # read new mapping
5646         my $map = {};
5647         my $statfile = "$tmpdir/qmrestore.stat";
5648         if (my $fd = IO::File->new($statfile, "r")) {
5649             while (defined (my $line = <$fd>)) {
5650                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5651                     $map->{$1} = $2 if $1;
5652                 } else {
5653                     print STDERR "unable to parse line in statfile - $line\n";
5654                 }
5655             }
5656             $fd->close();
5657         }
5658
5659         my $confsrc = "$tmpdir/qemu-server.conf";
5660
5661         my $srcfd = new IO::File($confsrc, "r") ||
5662             die "unable to open file '$confsrc'\n";
5663
5664         my $outfd = new IO::File ($tmpfn, "w") ||
5665             die "unable to write config for VM $vmid\n";
5666
5667         my $cookie = { netcount => 0 };
5668         while (defined (my $line = <$srcfd>)) {
5669             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5670         }
5671
5672         $srcfd->close();
5673         $outfd->close();
5674     };
5675     my $err = $@;
5676
5677     if ($err) {
5678
5679         unlink $tmpfn;
5680
5681         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5682
5683         die $err;
5684     }
5685
5686     rmtree $tmpdir;
5687
5688     rename $tmpfn, $conffile ||
5689         die "unable to commit configuration file '$conffile'\n";
5690
5691     PVE::Cluster::cfs_update(); # make sure we read new file
5692
5693     eval { rescan($vmid, 1); };
5694     warn $@ if $@;
5695 };
5696
5697 sub foreach_writable_storage {
5698     my ($conf, $func) = @_;
5699
5700     my $sidhash = {};
5701
5702     foreach my $ds (keys %$conf) {
5703         next if !is_valid_drivename($ds);
5704
5705         my $drive = parse_drive($ds, $conf->{$ds});
5706         next if !$drive;
5707         next if drive_is_cdrom($drive);
5708
5709         my $volid = $drive->{file};
5710
5711         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5712         $sidhash->{$sid} = $sid if $sid;
5713     }
5714
5715     foreach my $sid (sort keys %$sidhash) {
5716         &$func($sid);
5717     }
5718 }
5719
5720 sub do_snapshots_with_qemu {
5721     my ($storecfg, $volid) = @_;
5722
5723     my $storage_name = PVE::Storage::parse_volume_id($volid);
5724
5725     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5726         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5727         return 1;
5728     }
5729
5730     if ($volid =~ m/\.(qcow2|qed)$/){
5731         return 1;
5732     }
5733
5734     return undef;
5735 }
5736
5737 sub qga_check_running {
5738     my ($vmid) = @_;
5739
5740     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5741     if ($@) {
5742         warn "Qemu Guest Agent are not running - $@";
5743         return 0;
5744     }
5745     return 1;
5746 }
5747
5748 sub template_create {
5749     my ($vmid, $conf, $disk) = @_;
5750
5751     my $storecfg = PVE::Storage::config();
5752
5753     foreach_drive($conf, sub {
5754         my ($ds, $drive) = @_;
5755
5756         return if drive_is_cdrom($drive);
5757         return if $disk && $ds ne $disk;
5758
5759         my $volid = $drive->{file};
5760         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5761
5762         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5763         $drive->{file} = $voliddst;
5764         $conf->{$ds} = print_drive($vmid, $drive);
5765         PVE::QemuConfig->write_config($vmid, $conf);
5766     });
5767 }
5768
5769 sub qemu_img_convert {
5770     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5771
5772     my $storecfg = PVE::Storage::config();
5773     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5774     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5775
5776     if ($src_storeid && $dst_storeid) {
5777
5778         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5779
5780         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5781         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5782
5783         my $src_format = qemu_img_format($src_scfg, $src_volname);
5784         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5785
5786         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5787         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5788
5789         my $cmd = [];
5790         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5791         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5792         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5793         if ($is_zero_initialized) {
5794             push @$cmd, "zeroinit:$dst_path";
5795         } else {
5796             push @$cmd, $dst_path;
5797         }
5798
5799         my $parser = sub {
5800             my $line = shift;
5801             if($line =~ m/\((\S+)\/100\%\)/){
5802                 my $percent = $1;
5803                 my $transferred = int($size * $percent / 100);
5804                 my $remaining = $size - $transferred;
5805
5806                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5807             }
5808
5809         };
5810
5811         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5812         my $err = $@;
5813         die "copy failed: $err" if $err;
5814     }
5815 }
5816
5817 sub qemu_img_format {
5818     my ($scfg, $volname) = @_;
5819
5820     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5821         return $1;
5822     } else {
5823         return "raw";
5824     }
5825 }
5826
5827 sub qemu_drive_mirror {
5828     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5829
5830     my $storecfg = PVE::Storage::config();
5831     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5832
5833     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5834
5835     my $format = qemu_img_format($dst_scfg, $dst_volname);
5836
5837     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5838
5839     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5840
5841     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5842     $opts->{format} = $format if $format;
5843
5844     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5845
5846     my $finish_job = sub {
5847         while (1) {
5848             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5849             my $stat = @$stats[0];
5850             last if !$stat;
5851             sleep 1;
5852         }
5853     };
5854
5855     eval {
5856     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5857         while (1) {
5858             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5859             my $stat = @$stats[0];
5860             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5861             die "error job is not mirroring" if $stat->{type} ne "mirror";
5862
5863             my $busy = $stat->{busy};
5864             my $ready = $stat->{ready};
5865
5866             if (my $total = $stat->{len}) {
5867                 my $transferred = $stat->{offset} || 0;
5868                 my $remaining = $total - $transferred;
5869                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5870
5871                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5872             }
5873
5874
5875             if ($stat->{ready} eq 'true') {
5876
5877                 last if $vmiddst != $vmid;
5878
5879                 # try to switch the disk if source and destination are on the same guest
5880                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5881                 if (!$@) {
5882                     &$finish_job();
5883                     last;
5884                 }
5885                 die $@ if $@ !~ m/cannot be completed/;
5886             }
5887             sleep 1;
5888         }
5889
5890
5891     };
5892     my $err = $@;
5893
5894     my $cancel_job = sub {
5895         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5896         &$finish_job();
5897     };
5898
5899     if ($err) {
5900         eval { &$cancel_job(); };
5901         die "mirroring error: $err";
5902     }
5903
5904     if ($vmiddst != $vmid) {
5905         # if we clone a disk for a new target vm, we don't switch the disk
5906         &$cancel_job(); # so we call block-job-cancel
5907     }
5908 }
5909
5910 sub clone_disk {
5911     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5912         $newvmid, $storage, $format, $full, $newvollist) = @_;
5913
5914     my $newvolid;
5915
5916     if (!$full) {
5917         print "create linked clone of drive $drivename ($drive->{file})\n";
5918         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5919         push @$newvollist, $newvolid;
5920     } else {
5921         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5922         $storeid = $storage if $storage;
5923
5924         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5925         if (!$format) {
5926             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5927             $format = qemu_img_format($scfg, $volname);
5928         }
5929
5930         # test if requested format is supported - else use default
5931         my $supported = grep { $_ eq $format } @$validFormats;
5932         $format = $defFormat if !$supported;
5933
5934         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5935
5936         print "create full clone of drive $drivename ($drive->{file})\n";
5937         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5938         push @$newvollist, $newvolid;
5939
5940         PVE::Storage::activate_volumes($storecfg, $newvollist);
5941
5942         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5943         if (!$running || $snapname) {
5944             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5945         } else {
5946             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5947         }
5948     }
5949
5950     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5951
5952     my $disk = $drive;
5953     $disk->{format} = undef;
5954     $disk->{file} = $newvolid;
5955     $disk->{size} = $size;
5956
5957     return $disk;
5958 }
5959
5960 # this only works if VM is running
5961 sub get_current_qemu_machine {
5962     my ($vmid) = @_;
5963
5964     my $cmd = { execute => 'query-machines', arguments => {} };
5965     my $res = vm_qmp_command($vmid, $cmd);
5966
5967     my ($current, $default);
5968     foreach my $e (@$res) {
5969         $default = $e->{name} if $e->{'is-default'};
5970         $current = $e->{name} if $e->{'is-current'};
5971     }
5972
5973     # fallback to the default machine if current is not supported by qemu
5974     return $current || $default || 'pc';
5975 }
5976
5977 sub qemu_machine_feature_enabled {
5978     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5979
5980     my $current_major;
5981     my $current_minor;
5982
5983     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5984
5985         $current_major = $3;
5986         $current_minor = $4;
5987
5988     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
5989
5990         $current_major = $1;
5991         $current_minor = $2;
5992     }
5993
5994     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
5995
5996
5997 }
5998
5999 sub qemu_machine_pxe {
6000     my ($vmid, $conf, $machine) = @_;
6001
6002     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6003
6004     foreach my $opt (keys %$conf) {
6005         next if $opt !~ m/^net(\d+)$/;
6006         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6007         next if !$net;
6008         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6009         return $machine.".pxe" if $romfile =~ m/pxe/;
6010         last;
6011     }
6012
6013     return $machine;
6014 }
6015
6016 sub qemu_use_old_bios_files {
6017     my ($machine_type) = @_;
6018
6019     return if !$machine_type;
6020
6021     my $use_old_bios_files = undef;
6022
6023     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6024         $machine_type = $1;
6025         $use_old_bios_files = 1;
6026     } else {
6027         my $kvmver = kvm_user_version();
6028         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6029         # load new efi bios files on migration. So this hack is required to allow
6030         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6031         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6032         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6033     }
6034
6035     return ($use_old_bios_files, $machine_type);
6036 }
6037
6038 sub lspci {
6039
6040     my $devices = {};
6041
6042     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6043             my (undef, $id, $function) = @_;
6044             my $res = { id => $id, function => $function};
6045             push @{$devices->{$id}}, $res;
6046     });
6047
6048     return $devices;
6049 }
6050
6051 sub vm_iothreads_list {
6052     my ($vmid) = @_;
6053
6054     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6055
6056     my $iothreads = {};
6057     foreach my $iothread (@$res) {
6058         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6059     }
6060
6061     return $iothreads;
6062 }
6063
6064 sub scsihw_infos {
6065     my ($conf, $drive) = @_;
6066
6067     my $maxdev = 0;
6068
6069     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6070         $maxdev = 7;
6071     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6072         $maxdev = 1;
6073     } else {
6074         $maxdev = 256;
6075     }
6076
6077     my $controller = int($drive->{index} / $maxdev);
6078     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6079
6080     return ($maxdev, $controller, $controller_prefix);
6081 }
6082
6083 # bash completion helper
6084
6085 sub complete_backup_archives {
6086     my ($cmdname, $pname, $cvalue) = @_;
6087
6088     my $cfg = PVE::Storage::config();
6089
6090     my $storeid;
6091
6092     if ($cvalue =~ m/^([^:]+):/) {
6093         $storeid = $1;
6094     }
6095
6096     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6097
6098     my $res = [];
6099     foreach my $id (keys %$data) {
6100         foreach my $item (@{$data->{$id}}) {
6101             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6102             push @$res, $item->{volid} if defined($item->{volid});
6103         }
6104     }
6105
6106     return $res;
6107 }
6108
6109 my $complete_vmid_full = sub {
6110     my ($running) = @_;
6111
6112     my $idlist = vmstatus();
6113
6114     my $res = [];
6115
6116     foreach my $id (keys %$idlist) {
6117         my $d = $idlist->{$id};
6118         if (defined($running)) {
6119             next if $d->{template};
6120             next if $running && $d->{status} ne 'running';
6121             next if !$running && $d->{status} eq 'running';
6122         }
6123         push @$res, $id;
6124
6125     }
6126     return $res;
6127 };
6128
6129 sub complete_vmid {
6130     return &$complete_vmid_full();
6131 }
6132
6133 sub complete_vmid_stopped {
6134     return &$complete_vmid_full(0);
6135 }
6136
6137 sub complete_vmid_running {
6138     return &$complete_vmid_full(1);
6139 }
6140
6141 sub complete_storage {
6142
6143     my $cfg = PVE::Storage::config();
6144     my $ids = $cfg->{ids};
6145
6146     my $res = [];
6147     foreach my $sid (keys %$ids) {
6148         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6149         next if !$ids->{$sid}->{content}->{images};
6150         push @$res, $sid;
6151     }
6152
6153     return $res;
6154 }
6155
6156 1;