PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35 use URI::Escape;
  36
  37 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  38
  39 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  40
  41 # Note about locking: we use flock on the config file protect
  42 # against concurent actions.
  43 # Aditionaly, we have a 'lock' setting in the config file. This
  44 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  45 # allowed when such lock is set. But you can ignore this kind of
  46 # lock with the --skiplock flag.
  47
  48 cfs_register_file('/qemu-server/',
  49                   \&parse_vm_config,
  50                   \&write_vm_config);
  51
  52 PVE::JSONSchema::register_standard_option('skiplock', {
  53     description => "Ignore locks - only root is allowed to use this option.",
  54     type => 'boolean',
  55     optional => 1,
  56 });
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 #no warnings 'redefine';
  72
  73 sub cgroups_write {
  74    my ($controller, $vmid, $option, $value) = @_;
  75
  76    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  77    PVE::ProcFSTools::write_proc_entry($path, $value);
  78
  79 }
  80
  81 my $nodename = PVE::INotify::nodename();
  82
  83 mkdir "/etc/pve/nodes/$nodename";
  84 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  85 mkdir $confdir;
  86
  87 my $var_run_tmpdir = "/var/run/qemu-server";
  88 mkdir $var_run_tmpdir;
  89
  90 my $lock_dir = "/var/lock/qemu-server";
  91 mkdir $lock_dir;
  92
  93 my $pcisysfs = "/sys/bus/pci";
  94
  95 my $cpu_vendor_list = {
  96     # Intel CPUs
  97     486 => 'GenuineIntel',
  98     pentium => 'GenuineIntel',
  99     pentium2  => 'GenuineIntel',
 100     pentium3  => 'GenuineIntel',
 101     coreduo => 'GenuineIntel',
 102     core2duo => 'GenuineIntel',
 103     Conroe  => 'GenuineIntel',
 104     Penryn  => 'GenuineIntel',
 105     Nehalem  => 'GenuineIntel',
 106     Westmere => 'GenuineIntel',
 107     SandyBridge => 'GenuineIntel',
 108     IvyBridge => 'GenuineIntel',
 109     Haswell => 'GenuineIntel',
 110     'Haswell-noTSX' => 'GenuineIntel',
 111     Broadwell => 'GenuineIntel',
 112     'Broadwell-noTSX' => 'GenuineIntel',
 113
 114     # AMD CPUs
 115     athlon => 'AuthenticAMD',
 116     phenom  => 'AuthenticAMD',
 117     Opteron_G1  => 'AuthenticAMD',
 118     Opteron_G2  => 'AuthenticAMD',
 119     Opteron_G3  => 'AuthenticAMD',
 120     Opteron_G4  => 'AuthenticAMD',
 121     Opteron_G5  => 'AuthenticAMD',
 122
 123     # generic types, use vendor from host node
 124     host => 'default',
 125     kvm32 => 'default',
 126     kvm64 => 'default',
 127     qemu32 => 'default',
 128     qemu64 => 'default',
 129 };
 130
 131 my $cpu_fmt = {
 132     cputype => {
 133         description => "Emulated CPU type.",
 134         type => 'string',
 135         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 136         format_description => 'cputype',
 137         default => 'kvm64',
 138         default_key => 1,
 139     },
 140     hidden => {
 141         description => "Do not identify as a KVM virtual machine.",
 142         type => 'boolean',
 143         optional => 1,
 144         default => 0
 145     },
 146 };
 147
 148 my $watchdog_fmt = {
 149     model => {
 150         default_key => 1,
 151         type => 'string',
 152         enum => [qw(i6300esb ib700)],
 153         description => "Watchdog type to emulate.",
 154         default => 'i6300esb',
 155         optional => 1,
 156     },
 157     action => {
 158         type => 'string',
 159         enum => [qw(reset shutdown poweroff pause debug none)],
 160         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 161         optional => 1,
 162     },
 163 };
 164 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 165
 166 my $confdesc = {
 167     onboot => {
 168         optional => 1,
 169         type => 'boolean',
 170         description => "Specifies whether a VM will be started during system bootup.",
 171         default => 0,
 172     },
 173     autostart => {
 174         optional => 1,
 175         type => 'boolean',
 176         description => "Automatic restart after crash (currently ignored).",
 177         default => 0,
 178     },
 179     hotplug => {
 180         optional => 1,
 181         type => 'string', format => 'pve-hotplug-features',
 182         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 183         default => 'network,disk,usb',
 184     },
 185     reboot => {
 186         optional => 1,
 187         type => 'boolean',
 188         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 189         default => 1,
 190     },
 191     lock => {
 192         optional => 1,
 193         type => 'string',
 194         description => "Lock/unlock the VM.",
 195         enum => [qw(migrate backup snapshot rollback)],
 196     },
 197     cpulimit => {
 198         optional => 1,
 199         type => 'number',
 200         description => "Limit of CPU usage.",
 201         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 202         minimum => 0,
 203         maximum => 128,
 204         default => 0,
 205     },
 206     cpuunits => {
 207         optional => 1,
 208         type => 'integer',
 209         description => "CPU weight for a VM.",
 210         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 211         minimum => 0,
 212         maximum => 500000,
 213         default => 1000,
 214     },
 215     memory => {
 216         optional => 1,
 217         type => 'integer',
 218         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 219         minimum => 16,
 220         default => 512,
 221     },
 222     balloon => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 226         minimum => 0,
 227     },
 228     shares => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 232         minimum => 0,
 233         maximum => 50000,
 234         default => 1000,
 235     },
 236     keyboard => {
 237         optional => 1,
 238         type => 'string',
 239         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 240         enum => PVE::Tools::kvmkeymaplist(),
 241         default => 'en-us',
 242     },
 243     name => {
 244         optional => 1,
 245         type => 'string', format => 'dns-name',
 246         description => "Set a name for the VM. Only used on the configuration web interface.",
 247     },
 248     scsihw => {
 249         optional => 1,
 250         type => 'string',
 251         description => "SCSI controller model",
 252         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 253         default => 'lsi',
 254     },
 255     description => {
 256         optional => 1,
 257         type => 'string',
 258         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 259     },
 260     ostype => {
 261         optional => 1,
 262         type => 'string',
 263         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 264         description => "Specify guest operating system.",
 265         verbose_description => <<EODESC,
 266 Specify guest operating system. This is used to enable special
 267 optimization/features for specific operating systems:
 268
 269 [horizontal]
 270 other;; unspecified OS
 271 wxp;; Microsoft Windows XP
 272 w2k;; Microsoft Windows 2000
 273 w2k3;; Microsoft Windows 2003
 274 w2k8;; Microsoft Windows 2008
 275 wvista;; Microsoft Windows Vista
 276 win7;; Microsoft Windows 7
 277 win8;; Microsoft Windows 8/2012
 278 l24;; Linux 2.4 Kernel
 279 l26;; Linux 2.6/3.X Kernel
 280 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 281 EODESC
 282     },
 283     boot => {
 284         optional => 1,
 285         type => 'string',
 286         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 287         pattern => '[acdn]{1,4}',
 288         default => 'cdn',
 289     },
 290     bootdisk => {
 291         optional => 1,
 292         type => 'string', format => 'pve-qm-bootdisk',
 293         description => "Enable booting from specified disk.",
 294         pattern => '(ide|sata|scsi|virtio)\d+',
 295     },
 296     smp => {
 297         optional => 1,
 298         type => 'integer',
 299         description => "The number of CPUs. Please use option -sockets instead.",
 300         minimum => 1,
 301         default => 1,
 302     },
 303     sockets => {
 304         optional => 1,
 305         type => 'integer',
 306         description => "The number of CPU sockets.",
 307         minimum => 1,
 308         default => 1,
 309     },
 310     cores => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "The number of cores per socket.",
 314         minimum => 1,
 315         default => 1,
 316     },
 317     numa => {
 318         optional => 1,
 319         type => 'boolean',
 320         description => "Enable/disable NUMA.",
 321         default => 0,
 322     },
 323     vcpus => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Number of hotplugged vcpus.",
 327         minimum => 1,
 328         default => 0,
 329     },
 330     acpi => {
 331         optional => 1,
 332         type => 'boolean',
 333         description => "Enable/disable ACPI.",
 334         default => 1,
 335     },
 336     agent => {
 337         optional => 1,
 338         type => 'boolean',
 339         description => "Enable/disable Qemu GuestAgent.",
 340         default => 0,
 341     },
 342     kvm => {
 343         optional => 1,
 344         type => 'boolean',
 345         description => "Enable/disable KVM hardware virtualization.",
 346         default => 1,
 347     },
 348     tdf => {
 349         optional => 1,
 350         type => 'boolean',
 351         description => "Enable/disable time drift fix.",
 352         default => 0,
 353     },
 354     localtime => {
 355         optional => 1,
 356         type => 'boolean',
 357         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 358     },
 359     freeze => {
 360         optional => 1,
 361         type => 'boolean',
 362         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 363     },
 364     vga => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Select the VGA type.",
 368         verbose_description => "Select the VGA type. If you want to use high resolution" .
 369             " modes (>= 1280x1024x16) then you should use the options " .
 370             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 371             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 372             "display sever. For win* OS you can select how many independent " .
 373             "displays you want, Linux guests can add displays them self. " .
 374             "You can also run without any graphic card, using a serial device" .
 375             " as terminal.",
 376         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 377     },
 378     watchdog => {
 379         optional => 1,
 380         type => 'string', format => 'pve-qm-watchdog',
 381         description => "Create a virtual hardware watchdog device.",
 382         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 383             " (by a guest action), the watchdog must be periodically polled " .
 384             "by an agent inside the guest or else the watchdog will reset " .
 385             "the guest (or execute the respective action specified)",
 386     },
 387     startdate => {
 388         optional => 1,
 389         type => 'string',
 390         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 391         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 392         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 393         default => 'now',
 394     },
 395     startup =>  get_standard_option('pve-startup-order'),
 396     template => {
 397         optional => 1,
 398         type => 'boolean',
 399         description => "Enable/disable Template.",
 400         default => 0,
 401     },
 402     args => {
 403         optional => 1,
 404         type => 'string',
 405         description => "Arbitrary arguments passed to kvm.",
 406         verbose_description => <<EODESCR,
 407 Arbitrary arguments passed to kvm, for example:
 408
 409 args: -no-reboot -no-hpet
 410
 411 NOTE: this option is for experts only.
 412 EODESCR
 413     },
 414     tablet => {
 415         optional => 1,
 416         type => 'boolean',
 417         default => 1,
 418         description => "Enable/disable the USB tablet device.",
 419         verbose_description => "Enable/disable the USB tablet device. This device is " .
 420             "usually needed to allow absolute mouse positioning with VNC. " .
 421             "Else the mouse runs out of sync with normal VNC clients. " .
 422             "If you're running lots of console-only guests on one host, " .
 423             "you may consider disabling this to save some context switches. " .
 424             "This is turned off by default if you use spice (-vga=qxl).",
 425     },
 426     migrate_speed => {
 427         optional => 1,
 428         type => 'integer',
 429         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 430         minimum => 0,
 431         default => 0,
 432     },
 433     migrate_downtime => {
 434         optional => 1,
 435         type => 'number',
 436         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 437         minimum => 0,
 438         default => 0.1,
 439     },
 440     cdrom => {
 441         optional => 1,
 442         type => 'string', format => 'pve-qm-drive',
 443         typetext => 'volume',
 444         description => "This is an alias for option -ide2",
 445     },
 446     cpu => {
 447         optional => 1,
 448         description => "Emulated CPU type.",
 449         type => 'string',
 450         format => $cpu_fmt,
 451     },
 452     parent => get_standard_option('pve-snapshot-name', {
 453         optional => 1,
 454         description => "Parent snapshot name. This is used internally, and should not be modified.",
 455     }),
 456     snaptime => {
 457         optional => 1,
 458         description => "Timestamp for snapshots.",
 459         type => 'integer',
 460         minimum => 0,
 461     },
 462     vmstate => {
 463         optional => 1,
 464         type => 'string', format => 'pve-volume-id',
 465         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 466     },
 467     machine => {
 468         description => "Specific the Qemu machine type.",
 469         type => 'string',
 470         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 471         maxLength => 40,
 472         optional => 1,
 473     },
 474     smbios1 => {
 475         description => "Specify SMBIOS type 1 fields.",
 476         type => 'string', format => 'pve-qm-smbios1',
 477         maxLength => 256,
 478         optional => 1,
 479     },
 480     protection => {
 481         optional => 1,
 482         type => 'boolean',
 483         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 484         default => 0,
 485     },
 486     bios => {
 487         optional => 1,
 488         type => 'string',
 489         enum => [ qw(seabios ovmf) ],
 490         description => "Select BIOS implementation.",
 491         default => 'seabios',
 492     },
 493 };
 494
 495 # what about other qemu settings ?
 496 #cpu => 'string',
 497 #machine => 'string',
 498 #fda => 'file',
 499 #fdb => 'file',
 500 #mtdblock => 'file',
 501 #sd => 'file',
 502 #pflash => 'file',
 503 #snapshot => 'bool',
 504 #bootp => 'file',
 505 ##tftp => 'dir',
 506 ##smb => 'dir',
 507 #kernel => 'file',
 508 #append => 'string',
 509 #initrd => 'file',
 510 ##soundhw => 'string',
 511
 512 while (my ($k, $v) = each %$confdesc) {
 513     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 514 }
 515
 516 my $MAX_IDE_DISKS = 4;
 517 my $MAX_SCSI_DISKS = 14;
 518 my $MAX_VIRTIO_DISKS = 16;
 519 my $MAX_SATA_DISKS = 6;
 520 my $MAX_USB_DEVICES = 5;
 521 my $MAX_NETS = 32;
 522 my $MAX_UNUSED_DISKS = 8;
 523 my $MAX_HOSTPCI_DEVICES = 4;
 524 my $MAX_SERIAL_PORTS = 4;
 525 my $MAX_PARALLEL_PORTS = 3;
 526 my $MAX_NUMA = 8;
 527 my $MAX_MEM = 4194304;
 528 my $STATICMEM = 1024;
 529
 530 my $numa_fmt = {
 531     cpus => {
 532         type => "string",
 533         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 534         description => "CPUs accessing this NUMA node.",
 535         format_description => "id[-id];...",
 536     },
 537     memory => {
 538         type => "number",
 539         description => "Amount of memory this NUMA node provides.",
 540         optional => 1,
 541     },
 542     hostnodes => {
 543         type => "string",
 544         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 545         description => "Host NUMA nodes to use.",
 546         format_description => "id[-id];...",
 547         optional => 1,
 548     },
 549     policy => {
 550         type => 'string',
 551         enum => [qw(preferred bind interleave)],
 552         description => "NUMA allocation policy.",
 553         optional => 1,
 554     },
 555 };
 556 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 557 my $numadesc = {
 558     optional => 1,
 559     type => 'string', format => $numa_fmt,
 560     description => "NUMA topology.",
 561 };
 562 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 563
 564 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 565     $confdesc->{"numa$i"} = $numadesc;
 566 }
 567
 568 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 569                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 570                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 571 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 572
 573 my $net_fmt_bridge_descr = <<__EOD__;
 574 Bridge to attach the network device to. The Proxmox VE standard bridge
 575 is called 'vmbr0'.
 576
 577 If you do not specify a bridge, we create a kvm user (NATed) network
 578 device, which provides DHCP and DNS services. The following addresses
 579 are used:
 580
 581  10.0.2.2   Gateway
 582  10.0.2.3   DNS Server
 583  10.0.2.4   SMB Server
 584
 585 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 586 __EOD__
 587
 588 my $net_fmt = {
 589     macaddr => {
 590         type => 'string',
 591         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 592         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 593         format_description => "XX:XX:XX:XX:XX:XX",
 594         optional => 1,
 595     },
 596     model => {
 597         type => 'string',
 598         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 599         format_description => 'model',
 600         enum => $nic_model_list,
 601         default_key => 1,
 602     },
 603     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 604     bridge => {
 605         type => 'string',
 606         description => $net_fmt_bridge_descr,
 607         format_description => 'bridge',
 608         optional => 1,
 609     },
 610     queues => {
 611         type => 'integer',
 612         minimum => 0, maximum => 16,
 613         description => 'Number of packet queues to be used on the device.',
 614         optional => 1,
 615     },
 616     rate => {
 617         type => 'number',
 618         minimum => 0,
 619         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 620         optional => 1,
 621     },
 622     tag => {
 623         type => 'integer',
 624         minimum => 1, maximum => 4094,
 625         description => 'VLAN tag to apply to packets on this interface.',
 626         optional => 1,
 627     },
 628     trunks => {
 629         type => 'string',
 630         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 631         description => 'VLAN trunks to pass through this interface.',
 632         format_description => 'vlanid[;vlanid...]',
 633         optional => 1,
 634     },
 635     firewall => {
 636         type => 'boolean',
 637         description => 'Whether this interface should be protected by the firewall.',
 638         optional => 1,
 639     },
 640     link_down => {
 641         type => 'boolean',
 642         description => 'Whether this interface should be disconnected (like pulling the plug).',
 643         optional => 1,
 644     },
 645 };
 646
 647 my $netdesc = {
 648     optional => 1,
 649     type => 'string', format => $net_fmt,
 650     description => "Specify network devices.",
 651 };
 652
 653 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 654
 655 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 656     $confdesc->{"net$i"} = $netdesc;
 657 }
 658
 659 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 660 sub verify_volume_id_or_qm_path {
 661     my ($volid, $noerr) = @_;
 662
 663     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 664         return $volid;
 665     }
 666
 667     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 668     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 669     if ($@) {
 670         return undef if $noerr;
 671         die $@;
 672     }
 673     return $volid;
 674 }
 675
 676 my $drivename_hash;
 677
 678 my %drivedesc_base = (
 679     volume => { alias => 'file' },
 680     file => {
 681         type => 'string',
 682         format => 'pve-volume-id-or-qm-path',
 683         default_key => 1,
 684         format_description => 'volume',
 685         description => "The drive's backing volume.",
 686     },
 687     media => {
 688         type => 'string',
 689         enum => [qw(cdrom disk)],
 690         description => "The drive's media type.",
 691         default => 'disk',
 692         optional => 1
 693     },
 694     cyls => {
 695         type => 'integer',
 696         description => "Force the drive's physical geometry to have a specific cylinder count.",
 697         optional => 1
 698     },
 699     heads => {
 700         type => 'integer',
 701         description => "Force the drive's physical geometry to have a specific head count.",
 702         optional => 1
 703     },
 704     secs => {
 705         type => 'integer',
 706         description => "Force the drive's physical geometry to have a specific sector count.",
 707         optional => 1
 708     },
 709     trans => {
 710         type => 'string',
 711         enum => [qw(none lba auto)],
 712         description => "Force disk geometry bios translation mode.",
 713         optional => 1,
 714     },
 715     snapshot => {
 716         type => 'boolean',
 717         description => "Whether the drive should be included when making snapshots.",
 718         optional => 1,
 719     },
 720     cache => {
 721         type => 'string',
 722         enum => [qw(none writethrough writeback unsafe directsync)],
 723         description => "The drive's cache mode",
 724         optional => 1,
 725     },
 726     format => {
 727         type => 'string',
 728         format_description => 'image format',
 729         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 730         description => "The drive's backing file's data format.",
 731         optional => 1,
 732     },
 733     size => {
 734         type => 'string',
 735         format => 'disk-size',
 736         format_description => 'DiskSize',
 737         description => "Disk size. This is purely informational and has no effect.",
 738         optional => 1,
 739     },
 740     backup => {
 741         type => 'boolean',
 742         description => "Whether the drive should be included when making backups.",
 743         optional => 1,
 744     },
 745     werror => {
 746         type => 'string',
 747         enum => [qw(enospc ignore report stop)],
 748         description => 'Write error action.',
 749         optional => 1,
 750     },
 751     aio => {
 752         type => 'string',
 753         enum => [qw(native threads)],
 754         description => 'AIO type to use.',
 755         optional => 1,
 756     },
 757     discard => {
 758         type => 'string',
 759         enum => [qw(ignore on)],
 760         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 761         optional => 1,
 762     },
 763     detect_zeroes => {
 764         type => 'boolean',
 765         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 766         optional => 1,
 767     },
 768     serial => {
 769         type => 'string',
 770         format => 'urlencoded',
 771         format_description => 'serial',
 772         maxLength => 20*3, # *3 since it's %xx url enoded
 773         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 774         optional => 1,
 775     }
 776 );
 777
 778 my %rerror_fmt = (
 779     rerror => {
 780         type => 'string',
 781         enum => [qw(ignore report stop)],
 782         description => 'Read error action.',
 783         optional => 1,
 784     },
 785 );
 786
 787 my %iothread_fmt = ( iothread => {
 788         type => 'boolean',
 789         description => "Whether to use iothreads for this drive",
 790         optional => 1,
 791 });
 792
 793 my %model_fmt = (
 794     model => {
 795         type => 'string',
 796         format => 'urlencoded',
 797         format_description => 'model',
 798         maxLength => 40*3, # *3 since it's %xx url enoded
 799         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 800         optional => 1,
 801     },
 802 );
 803
 804 my %queues_fmt = (
 805     queues => {
 806         type => 'integer',
 807         description => "Number of queues.",
 808         minimum => 2,
 809         optional => 1
 810     }
 811 );
 812
 813 my $add_throttle_desc = sub {
 814     my ($key, $type, $what, $unit, $longunit) = @_;
 815     $drivedesc_base{$key} = {
 816         type => $type,
 817         format_description => $unit,
 818         description => "Maximum $what speed in $longunit per second.",
 819         optional => 1,
 820     };
 821 };
 822 # throughput: (leaky bucket)
 823 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 824 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 825 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 826 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 827 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 828 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 829 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 830 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 831 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 832
 833 # pools: (pool of IO before throttling starts taking effect)
 834 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 835 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 836 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 837 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 838 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 839 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 840
 841 my $ide_fmt = {
 842     %drivedesc_base,
 843     %rerror_fmt,
 844     %model_fmt,
 845 };
 846
 847 my $idedesc = {
 848     optional => 1,
 849     type => 'string', format => $ide_fmt,
 850     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 851 };
 852 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 853
 854 my $scsi_fmt = {
 855     %drivedesc_base,
 856     %iothread_fmt,
 857     %queues_fmt,
 858 };
 859 my $scsidesc = {
 860     optional => 1,
 861     type => 'string', format => $scsi_fmt,
 862     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 863 };
 864 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 865
 866 my $sata_fmt = {
 867     %drivedesc_base,
 868     %rerror_fmt,
 869 };
 870 my $satadesc = {
 871     optional => 1,
 872     type => 'string', format => $sata_fmt,
 873     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 874 };
 875 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 876
 877 my $virtio_fmt = {
 878     %drivedesc_base,
 879     %iothread_fmt,
 880     %rerror_fmt,
 881 };
 882 my $virtiodesc = {
 883     optional => 1,
 884     type => 'string', format => $virtio_fmt,
 885     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 886 };
 887 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 888
 889 my $alldrive_fmt = {
 890     %drivedesc_base,
 891     %rerror_fmt,
 892     %iothread_fmt,
 893     %model_fmt,
 894     %queues_fmt,
 895 };
 896
 897 my $usb_fmt = {
 898     host => {
 899         default_key => 1,
 900         type => 'string', format => 'pve-qm-usb-device',
 901         format_description => 'HOSTUSBDEVICE|spice',
 902         description => <<EODESCR,
 903 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 904
 905  'bus-port(.port)*' (decimal numbers) or
 906  'vendor_id:product_id' (hexadeciaml numbers) or
 907  'spice'
 908
 909 You can use the 'lsusb -t' command to list existing usb devices.
 910
 911 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 912
 913 The value 'spice' can be used to add a usb redirection devices for spice.
 914 EODESCR
 915     },
 916     usb3 => {
 917         optional => 1,
 918         type => 'boolean',
 919         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 920         default => 0,
 921     },
 922 };
 923
 924 my $usbdesc = {
 925     optional => 1,
 926     type => 'string', format => $usb_fmt,
 927     description => "Configure an USB device (n is 0 to 4).",
 928 };
 929 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 930
 931 # NOTE: the match-groups of this regex are used in parse_hostpci
 932 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 933 my $hostpci_fmt = {
 934     host => {
 935         default_key => 1,
 936         type => 'string',
 937         pattern => qr/$PCIRE(;$PCIRE)*/,
 938         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 939         description => <<EODESCR,
 940 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 941 of PCI virtual functions of the host. HOSTPCIID syntax is:
 942
 943 'bus:dev.func' (hexadecimal numbers)
 944
 945 You can us the 'lspci' command to list existing PCI devices.
 946
 947 NOTE: This option allows direct access to host hardware. So it is no longer
 948 possible to migrate such machines - use with special care.
 949
 950 CAUTION: Experimental! User reported problems with this option.
 951
 952 EODESCR
 953     },
 954     rombar => {
 955         type => 'boolean',
 956         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 957         optional => 1,
 958         default => 1,
 959     },
 960     pcie => {
 961         type => 'boolean',
 962         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
 963         optional => 1,
 964         default => 0,
 965     },
 966     'x-vga' => {
 967         type => 'boolean',
 968         description =>  "Enable vfio-vga device support.",
 969         optional => 1,
 970         default => 0,
 971     },
 972 };
 973 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
 974
 975 my $hostpcidesc = {
 976         optional => 1,
 977         type => 'string', format => 'pve-qm-hostpci',
 978         description => "Map host PCI devices into guest.",
 979 };
 980 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 981
 982 my $serialdesc = {
 983         optional => 1,
 984         type => 'string',
 985         pattern => '(/dev/.+|socket)',
 986         description =>  "Create a serial device inside the VM (n is 0 to 3)",
 987         verbose_description =>  <<EODESCR,
 988
 989 Create a serial device inside the VM (n is 0 to 3), and pass through a
 990 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
 991 host side (use 'qm terminal' to open a terminal connection).
 992
 993 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 994
 995 CAUTION: Experimental! User reported problems with this option.
 996 EODESCR
 997 };
 998
 999 my $paralleldesc= {
1000         optional => 1,
1001         type => 'string',
1002         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1003         description =>  "Map host parallel devices (n is 0 to 2).",
1004         verbose_description =>  <<EODESCR,
1005 Map host parallel devices (n is 0 to 2).
1006
1007 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1008
1009 CAUTION: Experimental! User reported problems with this option.
1010 EODESCR
1011 };
1012
1013 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1014     $confdesc->{"parallel$i"} = $paralleldesc;
1015 }
1016
1017 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1018     $confdesc->{"serial$i"} = $serialdesc;
1019 }
1020
1021 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1022     $confdesc->{"hostpci$i"} = $hostpcidesc;
1023 }
1024
1025 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1026     $drivename_hash->{"ide$i"} = 1;
1027     $confdesc->{"ide$i"} = $idedesc;
1028 }
1029
1030 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1031     $drivename_hash->{"sata$i"} = 1;
1032     $confdesc->{"sata$i"} = $satadesc;
1033 }
1034
1035 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1036     $drivename_hash->{"scsi$i"} = 1;
1037     $confdesc->{"scsi$i"} = $scsidesc ;
1038 }
1039
1040 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1041     $drivename_hash->{"virtio$i"} = 1;
1042     $confdesc->{"virtio$i"} = $virtiodesc;
1043 }
1044
1045 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1046     $confdesc->{"usb$i"} = $usbdesc;
1047 }
1048
1049 my $unuseddesc = {
1050     optional => 1,
1051     type => 'string', format => 'pve-volume-id',
1052     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1053 };
1054
1055 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1056     $confdesc->{"unused$i"} = $unuseddesc;
1057 }
1058
1059 my $kvm_api_version = 0;
1060
1061 sub kvm_version {
1062
1063     return $kvm_api_version if $kvm_api_version;
1064
1065     my $fh = IO::File->new("</dev/kvm") ||
1066         return 0;
1067
1068     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1069         $kvm_api_version = $v;
1070     }
1071
1072     $fh->close();
1073
1074     return  $kvm_api_version;
1075 }
1076
1077 my $kvm_user_version;
1078
1079 sub kvm_user_version {
1080
1081     return $kvm_user_version if $kvm_user_version;
1082
1083     $kvm_user_version = 'unknown';
1084
1085     my $code = sub {
1086         my $line = shift;
1087         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1088             $kvm_user_version = $2;
1089         }
1090     };
1091
1092     eval { run_command("kvm -version", outfunc => $code); };
1093     warn $@ if $@;
1094
1095     return $kvm_user_version;
1096
1097 }
1098
1099 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1100
1101 sub valid_drive_names {
1102     # order is important - used to autoselect boot disk
1103     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1104             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1105             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1106             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1107 }
1108
1109 sub is_valid_drivename {
1110     my $dev = shift;
1111
1112     return defined($drivename_hash->{$dev});
1113 }
1114
1115 sub option_exists {
1116     my $key = shift;
1117     return defined($confdesc->{$key});
1118 }
1119
1120 sub nic_models {
1121     return $nic_model_list;
1122 }
1123
1124 sub os_list_description {
1125
1126     return {
1127         other => 'Other',
1128         wxp => 'Windows XP',
1129         w2k => 'Windows 2000',
1130         w2k3 =>, 'Windows 2003',
1131         w2k8 => 'Windows 2008',
1132         wvista => 'Windows Vista',
1133         win7 => 'Windows 7',
1134         win8 => 'Windows 8/2012',
1135         l24 => 'Linux 2.4',
1136         l26 => 'Linux 2.6',
1137     };
1138 }
1139
1140 my $cdrom_path;
1141
1142 sub get_cdrom_path {
1143
1144     return  $cdrom_path if $cdrom_path;
1145
1146     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1147     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1148     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1149 }
1150
1151 sub get_iso_path {
1152     my ($storecfg, $vmid, $cdrom) = @_;
1153
1154     if ($cdrom eq 'cdrom') {
1155         return get_cdrom_path();
1156     } elsif ($cdrom eq 'none') {
1157         return '';
1158     } elsif ($cdrom =~ m|^/|) {
1159         return $cdrom;
1160     } else {
1161         return PVE::Storage::path($storecfg, $cdrom);
1162     }
1163 }
1164
1165 # try to convert old style file names to volume IDs
1166 sub filename_to_volume_id {
1167     my ($vmid, $file, $media) = @_;
1168
1169     if (!($file eq 'none' || $file eq 'cdrom' ||
1170           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1171
1172         return undef if $file =~ m|/|;
1173
1174         if ($media && $media eq 'cdrom') {
1175             $file = "local:iso/$file";
1176         } else {
1177             $file = "local:$vmid/$file";
1178         }
1179     }
1180
1181     return $file;
1182 }
1183
1184 sub verify_media_type {
1185     my ($opt, $vtype, $media) = @_;
1186
1187     return if !$media;
1188
1189     my $etype;
1190     if ($media eq 'disk') {
1191         $etype = 'images';
1192     } elsif ($media eq 'cdrom') {
1193         $etype = 'iso';
1194     } else {
1195         die "internal error";
1196     }
1197
1198     return if ($vtype eq $etype);
1199
1200     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1201 }
1202
1203 sub cleanup_drive_path {
1204     my ($opt, $storecfg, $drive) = @_;
1205
1206     # try to convert filesystem paths to volume IDs
1207
1208     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1209         ($drive->{file} !~ m|^/dev/.+|) &&
1210         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1211         ($drive->{file} !~ m/^\d+$/)) {
1212         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1213         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1214         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1215         verify_media_type($opt, $vtype, $drive->{media});
1216         $drive->{file} = $volid;
1217     }
1218
1219     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1220 }
1221
1222 sub parse_hotplug_features {
1223     my ($data) = @_;
1224
1225     my $res = {};
1226
1227     return $res if $data eq '0';
1228
1229     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1230
1231     foreach my $feature (PVE::Tools::split_list($data)) {
1232         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1233             $res->{$1} = 1;
1234         } else {
1235             warn "ignoring unknown hotplug feature '$feature'\n";
1236         }
1237     }
1238     return $res;
1239 }
1240
1241 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1242 sub pve_verify_hotplug_features {
1243     my ($value, $noerr) = @_;
1244
1245     return $value if parse_hotplug_features($value);
1246
1247     return undef if $noerr;
1248
1249     die "unable to parse hotplug option\n";
1250 }
1251
1252 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1253 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1254 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1255 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1256 #        [,iothread=on][,serial=serial][,model=model]
1257
1258 sub parse_drive {
1259     my ($key, $data) = @_;
1260
1261     my ($interface, $index);
1262
1263     if ($key =~ m/^([^\d]+)(\d+)$/) {
1264         $interface = $1;
1265         $index = $2;
1266     } else {
1267         return undef;
1268     }
1269
1270     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1271                                      : $confdesc->{$key}->{format};
1272     if (!$desc) {
1273         warn "invalid drive key: $key\n";
1274         return undef;
1275     }
1276     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1277     return undef if !$res;
1278     $res->{interface} = $interface;
1279     $res->{index} = $index;
1280
1281     my $error = 0;
1282     foreach my $opt (qw(bps bps_rd bps_wr)) {
1283         if (my $bps = defined(delete $res->{$opt})) {
1284             if (defined($res->{"m$opt"})) {
1285                 warn "both $opt and m$opt specified\n";
1286                 ++$error;
1287                 next;
1288             }
1289             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1290         }
1291     }
1292     return undef if $error;
1293
1294     return undef if $res->{mbps_rd} && $res->{mbps};
1295     return undef if $res->{mbps_wr} && $res->{mbps};
1296     return undef if $res->{iops_rd} && $res->{iops};
1297     return undef if $res->{iops_wr} && $res->{iops};
1298
1299     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1300         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1301         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1302         return undef if $res->{interface} eq 'virtio';
1303     }
1304
1305     if (my $size = $res->{size}) {
1306         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1307     }
1308
1309     return $res;
1310 }
1311
1312 sub print_drive {
1313     my ($vmid, $drive) = @_;
1314     my $data = { %$drive };
1315     delete $data->{$_} for qw(index interface);
1316     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1317 }
1318
1319 sub scsi_inquiry {
1320     my($fh, $noerr) = @_;
1321
1322     my $SG_IO = 0x2285;
1323     my $SG_GET_VERSION_NUM = 0x2282;
1324
1325     my $versionbuf = "\x00" x 8;
1326     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1327     if (!$ret) {
1328         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1329         return undef;
1330     }
1331     my $version = unpack("I", $versionbuf);
1332     if ($version < 30000) {
1333         die "scsi generic interface too old\n"  if !$noerr;
1334         return undef;
1335     }
1336
1337     my $buf = "\x00" x 36;
1338     my $sensebuf = "\x00" x 8;
1339     my $cmd = pack("C x3 C x1", 0x12, 36);
1340
1341     # see /usr/include/scsi/sg.h
1342     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1343
1344     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1345                       length($sensebuf), 0, length($buf), $buf,
1346                       $cmd, $sensebuf, 6000);
1347
1348     $ret = ioctl($fh, $SG_IO, $packet);
1349     if (!$ret) {
1350         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1351         return undef;
1352     }
1353
1354     my @res = unpack($sg_io_hdr_t, $packet);
1355     if ($res[17] || $res[18]) {
1356         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1357         return undef;
1358     }
1359
1360     my $res = {};
1361     (my $byte0, my $byte1, $res->{vendor},
1362      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1363
1364     $res->{removable} = $byte1 & 128 ? 1 : 0;
1365     $res->{type} = $byte0 & 31;
1366
1367     return $res;
1368 }
1369
1370 sub path_is_scsi {
1371     my ($path) = @_;
1372
1373     my $fh = IO::File->new("+<$path") || return undef;
1374     my $res = scsi_inquiry($fh, 1);
1375     close($fh);
1376
1377     return $res;
1378 }
1379
1380 sub machine_type_is_q35 {
1381     my ($conf) = @_;
1382
1383     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1384 }
1385
1386 sub print_tabletdevice_full {
1387     my ($conf) = @_;
1388
1389     my $q35 = machine_type_is_q35($conf);
1390
1391     # we use uhci for old VMs because tablet driver was buggy in older qemu
1392     my $usbbus = $q35 ? "ehci" : "uhci";
1393
1394     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1395 }
1396
1397 sub print_drivedevice_full {
1398     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1399
1400     my $device = '';
1401     my $maxdev = 0;
1402
1403     if ($drive->{interface} eq 'virtio') {
1404         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1405         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1406         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1407     } elsif ($drive->{interface} eq 'scsi') {
1408
1409         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1410         my $unit = $drive->{index} % $maxdev;
1411         my $devicetype = 'hd';
1412         my $path = '';
1413         if (drive_is_cdrom($drive)) {
1414             $devicetype = 'cd';
1415         } else {
1416             if ($drive->{file} =~ m|^/|) {
1417                 $path = $drive->{file};
1418                 if (my $info = path_is_scsi($path)) {
1419                     if ($info->{type} == 0) {
1420                         $devicetype = 'block';
1421                     } elsif ($info->{type} == 1) { # tape
1422                         $devicetype = 'generic';
1423                     }
1424                 }
1425             } else {
1426                  $path = PVE::Storage::path($storecfg, $drive->{file});
1427             }
1428
1429             if($path =~ m/^iscsi\:\/\//){
1430                 $devicetype = 'generic';
1431             }
1432         }
1433
1434         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1435             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1436         } else {
1437             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1438         }
1439
1440     } elsif ($drive->{interface} eq 'ide'){
1441         $maxdev = 2;
1442         my $controller = int($drive->{index} / $maxdev);
1443         my $unit = $drive->{index} % $maxdev;
1444         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1445
1446         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1447         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1448             $model = URI::Escape::uri_unescape($model);
1449             $device .= ",model=$model";
1450         }
1451     } elsif ($drive->{interface} eq 'sata'){
1452         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1453         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1454         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1455     } elsif ($drive->{interface} eq 'usb') {
1456         die "implement me";
1457         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1458     } else {
1459         die "unsupported interface type";
1460     }
1461
1462     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1463
1464     return $device;
1465 }
1466
1467 sub get_initiator_name {
1468     my $initiator;
1469
1470     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1471     while (defined(my $line = <$fh>)) {
1472         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1473         $initiator = $1;
1474         last;
1475     }
1476     $fh->close();
1477
1478     return $initiator;
1479 }
1480
1481 sub print_drive_full {
1482     my ($storecfg, $vmid, $drive) = @_;
1483
1484     my $path;
1485     my $volid = $drive->{file};
1486     my $format;
1487
1488     if (drive_is_cdrom($drive)) {
1489         $path = get_iso_path($storecfg, $vmid, $volid);
1490     } else {
1491         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1492         if ($storeid) {
1493             $path = PVE::Storage::path($storecfg, $volid);
1494             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1495             $format = qemu_img_format($scfg, $volname);
1496         } else {
1497             $path = $volid;
1498             $format = "raw";
1499         }
1500    }
1501
1502     my $opts = '';
1503     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1504     foreach my $o (@qemu_drive_options) {
1505         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1506     }
1507     if (my $serial = $drive->{serial}) {
1508         $serial = URI::Escape::uri_unescape($serial);
1509         $opts .= ",serial=$serial";
1510     }
1511
1512     $opts .= ",format=$format" if $format && !$drive->{format};
1513
1514     foreach my $o (qw(bps bps_rd bps_wr)) {
1515         my $v = $drive->{"m$o"};
1516         $opts .= ",$o=" . int($v*1024*1024) if $v;
1517     }
1518
1519     my $cache_direct = 0;
1520
1521     if (my $cache = $drive->{cache}) {
1522         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1523     } elsif (!drive_is_cdrom($drive)) {
1524         $opts .= ",cache=none";
1525         $cache_direct = 1;
1526     }
1527
1528     # aio native works only with O_DIRECT
1529     if (!$drive->{aio}) {
1530         if($cache_direct) {
1531             $opts .= ",aio=native";
1532         } else {
1533             $opts .= ",aio=threads";
1534         }
1535     }
1536
1537     if (!drive_is_cdrom($drive)) {
1538         my $detectzeroes;
1539         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1540             $detectzeroes = 'off';
1541         } elsif ($drive->{discard}) {
1542             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1543         } else {
1544             # This used to be our default with discard not being specified:
1545             $detectzeroes = 'on';
1546         }
1547         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1548     }
1549
1550     my $pathinfo = $path ? "file=$path," : '';
1551
1552     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1553 }
1554
1555 sub print_netdevice_full {
1556     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1557
1558     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1559
1560     my $device = $net->{model};
1561     if ($net->{model} eq 'virtio') {
1562          $device = 'virtio-net-pci';
1563      };
1564
1565     my $pciaddr = print_pci_addr("$netid", $bridges);
1566     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1567     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1568         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1569         my $vectors = $net->{queues} * 2 + 2;
1570         $tmpstr .= ",vectors=$vectors,mq=on";
1571     }
1572     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1573
1574     if ($use_old_bios_files) {
1575         my $romfile;
1576         if ($device eq 'virtio-net-pci') {
1577             $romfile = 'pxe-virtio.rom';
1578         } elsif ($device eq 'e1000') {
1579             $romfile = 'pxe-e1000.rom';
1580         } elsif ($device eq 'ne2k') {
1581             $romfile = 'pxe-ne2k_pci.rom';
1582         } elsif ($device eq 'pcnet') {
1583             $romfile = 'pxe-pcnet.rom';
1584         } elsif ($device eq 'rtl8139') {
1585             $romfile = 'pxe-rtl8139.rom';
1586         }
1587         $tmpstr .= ",romfile=$romfile" if $romfile;
1588     }
1589
1590     return $tmpstr;
1591 }
1592
1593 sub print_netdev_full {
1594     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1595
1596     my $i = '';
1597     if ($netid =~ m/^net(\d+)$/) {
1598         $i = int($1);
1599     }
1600
1601     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1602
1603     my $ifname = "tap${vmid}i$i";
1604
1605     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1606     die "interface name '$ifname' is too long (max 15 character)\n"
1607         if length($ifname) >= 16;
1608
1609     my $vhostparam = '';
1610     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1611
1612     my $vmname = $conf->{name} || "vm$vmid";
1613
1614     my $netdev = "";
1615     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1616
1617     if ($net->{bridge}) {
1618         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1619     } else {
1620         $netdev = "type=user,id=$netid,hostname=$vmname";
1621     }
1622
1623     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1624
1625     return $netdev;
1626 }
1627
1628 sub drive_is_cdrom {
1629     my ($drive) = @_;
1630
1631     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1632
1633 }
1634
1635 sub parse_number_sets {
1636     my ($set) = @_;
1637     my $res = [];
1638     foreach my $part (split(/;/, $set)) {
1639         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1640             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1641             push @$res, [ $1, $2 ];
1642         } else {
1643             die "invalid range: $part\n";
1644         }
1645     }
1646     return $res;
1647 }
1648
1649 sub parse_numa {
1650     my ($data) = @_;
1651
1652     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1653     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1654     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1655     return $res;
1656 }
1657
1658 sub parse_hostpci {
1659     my ($value) = @_;
1660
1661     return undef if !$value;
1662
1663     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1664
1665     my @idlist = split(/;/, $res->{host});
1666     delete $res->{host};
1667     foreach my $id (@idlist) {
1668         if ($id =~ /^$PCIRE$/) {
1669             push @{$res->{pciid}}, { id => $1, function => ($2//'0') };
1670         } else {
1671             # should have been caught by parse_property_string already
1672             die "failed to parse PCI id: $id\n";
1673         }
1674     }
1675     return $res;
1676 }
1677
1678 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1679 sub parse_net {
1680     my ($data) = @_;
1681
1682     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1683     if ($@) {
1684         warn $@;
1685         return undef;
1686     }
1687     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1688     return $res;
1689 }
1690
1691 sub print_net {
1692     my $net = shift;
1693
1694     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1695 }
1696
1697 sub add_random_macs {
1698     my ($settings) = @_;
1699
1700     foreach my $opt (keys %$settings) {
1701         next if $opt !~ m/^net(\d+)$/;
1702         my $net = parse_net($settings->{$opt});
1703         next if !$net;
1704         $settings->{$opt} = print_net($net);
1705     }
1706 }
1707
1708 sub vm_is_volid_owner {
1709     my ($storecfg, $vmid, $volid) = @_;
1710
1711     if ($volid !~  m|^/|) {
1712         my ($path, $owner);
1713         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1714         if ($owner && ($owner == $vmid)) {
1715             return 1;
1716         }
1717     }
1718
1719     return undef;
1720 }
1721
1722 sub split_flagged_list {
1723     my $text = shift || '';
1724     $text =~ s/[,;]/ /g;
1725     $text =~ s/^\s+//;
1726     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1727 }
1728
1729 sub join_flagged_list {
1730     my ($how, $lst) = @_;
1731     join $how, map { $lst->{$_} . $_ } keys %$lst;
1732 }
1733
1734 sub vmconfig_delete_pending_option {
1735     my ($conf, $key, $force) = @_;
1736
1737     delete $conf->{pending}->{$key};
1738     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1739     $pending_delete_hash->{$key} = $force ? '!' : '';
1740     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1741 }
1742
1743 sub vmconfig_undelete_pending_option {
1744     my ($conf, $key) = @_;
1745
1746     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1747     delete $pending_delete_hash->{$key};
1748
1749     if (%$pending_delete_hash) {
1750         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1751     } else {
1752         delete $conf->{pending}->{delete};
1753     }
1754 }
1755
1756 sub vmconfig_register_unused_drive {
1757     my ($storecfg, $vmid, $conf, $drive) = @_;
1758
1759     if (!drive_is_cdrom($drive)) {
1760         my $volid = $drive->{file};
1761         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1762             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1763         }
1764     }
1765 }
1766
1767 sub vmconfig_cleanup_pending {
1768     my ($conf) = @_;
1769
1770     # remove pending changes when nothing changed
1771     my $changes;
1772     foreach my $opt (keys %{$conf->{pending}}) {
1773         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1774             $changes = 1;
1775             delete $conf->{pending}->{$opt};
1776         }
1777     }
1778
1779     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1780     my $pending_delete_hash = {};
1781     while (my ($opt, $force) = each %$current_delete_hash) {
1782         if (defined($conf->{$opt})) {
1783             $pending_delete_hash->{$opt} = $force;
1784         } else {
1785             $changes = 1;
1786         }
1787     }
1788
1789     if (%$pending_delete_hash) {
1790         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1791     } else {
1792         delete $conf->{pending}->{delete};
1793     }
1794
1795     return $changes;
1796 }
1797
1798 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1799 my $smbios1_fmt = {
1800     uuid => {
1801         type => 'string',
1802         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1803         format_description => 'UUID',
1804         description => "Set SMBIOS1 UUID.",
1805         optional => 1,
1806     },
1807     version => {
1808         type => 'string',
1809         pattern => '\S+',
1810         format_description => 'string',
1811         description => "Set SMBIOS1 version.",
1812         optional => 1,
1813     },
1814     serial => {
1815         type => 'string',
1816         pattern => '\S+',
1817         format_description => 'string',
1818         description => "Set SMBIOS1 serial number.",
1819         optional => 1,
1820     },
1821     manufacturer => {
1822         type => 'string',
1823         pattern => '\S+',
1824         format_description => 'string',
1825         description => "Set SMBIOS1 manufacturer.",
1826         optional => 1,
1827     },
1828     product => {
1829         type => 'string',
1830         pattern => '\S+',
1831         format_description => 'string',
1832         description => "Set SMBIOS1 product ID.",
1833         optional => 1,
1834     },
1835     sku => {
1836         type => 'string',
1837         pattern => '\S+',
1838         format_description => 'string',
1839         description => "Set SMBIOS1 SKU string.",
1840         optional => 1,
1841     },
1842     family => {
1843         type => 'string',
1844         pattern => '\S+',
1845         format_description => 'string',
1846         description => "Set SMBIOS1 family string.",
1847         optional => 1,
1848     },
1849 };
1850
1851 sub parse_smbios1 {
1852     my ($data) = @_;
1853
1854     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1855     warn $@ if $@;
1856     return $res;
1857 }
1858
1859 sub print_smbios1 {
1860     my ($smbios1) = @_;
1861     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1862 }
1863
1864 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1865
1866 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1867 sub verify_bootdisk {
1868     my ($value, $noerr) = @_;
1869
1870     return $value if is_valid_drivename($value);
1871
1872     return undef if $noerr;
1873
1874     die "invalid boot disk '$value'\n";
1875 }
1876
1877 sub parse_watchdog {
1878     my ($value) = @_;
1879
1880     return undef if !$value;
1881
1882     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1883     warn $@ if $@;
1884     return $res;
1885 }
1886
1887 sub parse_usb_device {
1888     my ($value) = @_;
1889
1890     return undef if !$value;
1891
1892     my $res = {};
1893     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1894         $res->{vendorid} = $2;
1895         $res->{productid} = $4;
1896     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1897         $res->{hostbus} = $1;
1898         $res->{hostport} = $2;
1899     } elsif ($value =~ m/^spice$/i) {
1900         $res->{spice} = 1;
1901     } else {
1902         return undef;
1903     }
1904
1905     return $res;
1906 }
1907
1908 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1909 sub verify_usb_device {
1910     my ($value, $noerr) = @_;
1911
1912     return $value if parse_usb_device($value);
1913
1914     return undef if $noerr;
1915
1916     die "unable to parse usb device\n";
1917 }
1918
1919 # add JSON properties for create and set function
1920 sub json_config_properties {
1921     my $prop = shift;
1922
1923     foreach my $opt (keys %$confdesc) {
1924         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1925         $prop->{$opt} = $confdesc->{$opt};
1926     }
1927
1928     return $prop;
1929 }
1930
1931 sub check_type {
1932     my ($key, $value) = @_;
1933
1934     die "unknown setting '$key'\n" if !$confdesc->{$key};
1935
1936     my $type = $confdesc->{$key}->{type};
1937
1938     if (!defined($value)) {
1939         die "got undefined value\n";
1940     }
1941
1942     if ($value =~ m/[\n\r]/) {
1943         die "property contains a line feed\n";
1944     }
1945
1946     if ($type eq 'boolean') {
1947         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1948         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1949         die "type check ('boolean') failed - got '$value'\n";
1950     } elsif ($type eq 'integer') {
1951         return int($1) if $value =~ m/^(\d+)$/;
1952         die "type check ('integer') failed - got '$value'\n";
1953     } elsif ($type eq 'number') {
1954         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1955         die "type check ('number') failed - got '$value'\n";
1956     } elsif ($type eq 'string') {
1957         if (my $fmt = $confdesc->{$key}->{format}) {
1958             if ($fmt eq 'pve-qm-drive') {
1959                 # special case - we need to pass $key to parse_drive()
1960                 my $drive = parse_drive($key, $value);
1961                 return $value if $drive;
1962                 die "unable to parse drive options\n";
1963             }
1964             PVE::JSONSchema::check_format($fmt, $value);
1965             return $value;
1966         }
1967         $value =~ s/^\"(.*)\"$/$1/;
1968         return $value;
1969     } else {
1970         die "internal error"
1971     }
1972 }
1973
1974 sub check_iommu_support{
1975     #fixme : need to check IOMMU support
1976     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1977
1978     my $iommu=1;
1979     return $iommu;
1980
1981 }
1982
1983 sub touch_config {
1984     my ($vmid) = @_;
1985
1986     my $conf = PVE::QemuConfig->config_file($vmid);
1987     utime undef, undef, $conf;
1988 }
1989
1990 sub destroy_vm {
1991     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1992
1993     my $conffile = PVE::QemuConfig->config_file($vmid);
1994
1995     my $conf = PVE::QemuConfig->load_config($vmid);
1996
1997     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1998
1999     # only remove disks owned by this VM
2000     foreach_drive($conf, sub {
2001         my ($ds, $drive) = @_;
2002
2003         return if drive_is_cdrom($drive);
2004
2005         my $volid = $drive->{file};
2006
2007         return if !$volid || $volid =~ m|^/|;
2008
2009         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2010         return if !$path || !$owner || ($owner != $vmid);
2011
2012         PVE::Storage::vdisk_free($storecfg, $volid);
2013     });
2014
2015     if ($keep_empty_config) {
2016         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2017     } else {
2018         unlink $conffile;
2019     }
2020
2021     # also remove unused disk
2022     eval {
2023         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2024
2025         eval {
2026             PVE::Storage::foreach_volid($dl, sub {
2027                 my ($volid, $sid, $volname, $d) = @_;
2028                 PVE::Storage::vdisk_free($storecfg, $volid);
2029             });
2030         };
2031         warn $@ if $@;
2032
2033     };
2034     warn $@ if $@;
2035 }
2036
2037 sub parse_vm_config {
2038     my ($filename, $raw) = @_;
2039
2040     return undef if !defined($raw);
2041
2042     my $res = {
2043         digest => Digest::SHA::sha1_hex($raw),
2044         snapshots => {},
2045         pending => {},
2046     };
2047
2048     $filename =~ m|/qemu-server/(\d+)\.conf$|
2049         || die "got strange filename '$filename'";
2050
2051     my $vmid = $1;
2052
2053     my $conf = $res;
2054     my $descr;
2055     my $section = '';
2056
2057     my @lines = split(/\n/, $raw);
2058     foreach my $line (@lines) {
2059         next if $line =~ m/^\s*$/;
2060
2061         if ($line =~ m/^\[PENDING\]\s*$/i) {
2062             $section = 'pending';
2063             if (defined($descr)) {
2064                 $descr =~ s/\s+$//;
2065                 $conf->{description} = $descr;
2066             }
2067             $descr = undef;
2068             $conf = $res->{$section} = {};
2069             next;
2070
2071         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2072             $section = $1;
2073             if (defined($descr)) {
2074                 $descr =~ s/\s+$//;
2075                 $conf->{description} = $descr;
2076             }
2077             $descr = undef;
2078             $conf = $res->{snapshots}->{$section} = {};
2079             next;
2080         }
2081
2082         if ($line =~ m/^\#(.*)\s*$/) {
2083             $descr = '' if !defined($descr);
2084             $descr .= PVE::Tools::decode_text($1) . "\n";
2085             next;
2086         }
2087
2088         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2089             $descr = '' if !defined($descr);
2090             $descr .= PVE::Tools::decode_text($2);
2091         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2092             $conf->{snapstate} = $1;
2093         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2094             my $key = $1;
2095             my $value = $2;
2096             $conf->{$key} = $value;
2097         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2098             my $value = $1;
2099             if ($section eq 'pending') {
2100                 $conf->{delete} = $value; # we parse this later
2101             } else {
2102                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2103             }
2104         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2105             my $key = $1;
2106             my $value = $2;
2107             eval { $value = check_type($key, $value); };
2108             if ($@) {
2109                 warn "vm $vmid - unable to parse value of '$key' - $@";
2110             } else {
2111                 my $fmt = $confdesc->{$key}->{format};
2112                 if ($fmt && $fmt eq 'pve-qm-drive') {
2113                     my $v = parse_drive($key, $value);
2114                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2115                         $v->{file} = $volid;
2116                         $value = print_drive($vmid, $v);
2117                     } else {
2118                         warn "vm $vmid - unable to parse value of '$key'\n";
2119                         next;
2120                     }
2121                 }
2122
2123                 if ($key eq 'cdrom') {
2124                     $conf->{ide2} = $value;
2125                 } else {
2126                     $conf->{$key} = $value;
2127                 }
2128             }
2129         }
2130     }
2131
2132     if (defined($descr)) {
2133         $descr =~ s/\s+$//;
2134         $conf->{description} = $descr;
2135     }
2136     delete $res->{snapstate}; # just to be sure
2137
2138     return $res;
2139 }
2140
2141 sub write_vm_config {
2142     my ($filename, $conf) = @_;
2143
2144     delete $conf->{snapstate}; # just to be sure
2145
2146     if ($conf->{cdrom}) {
2147         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2148         $conf->{ide2} = $conf->{cdrom};
2149         delete $conf->{cdrom};
2150     }
2151
2152     # we do not use 'smp' any longer
2153     if ($conf->{sockets}) {
2154         delete $conf->{smp};
2155     } elsif ($conf->{smp}) {
2156         $conf->{sockets} = $conf->{smp};
2157         delete $conf->{cores};
2158         delete $conf->{smp};
2159     }
2160
2161     my $used_volids = {};
2162
2163     my $cleanup_config = sub {
2164         my ($cref, $pending, $snapname) = @_;
2165
2166         foreach my $key (keys %$cref) {
2167             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2168                 $key eq 'snapstate' || $key eq 'pending';
2169             my $value = $cref->{$key};
2170             if ($key eq 'delete') {
2171                 die "propertry 'delete' is only allowed in [PENDING]\n"
2172                     if !$pending;
2173                 # fixme: check syntax?
2174                 next;
2175             }
2176             eval { $value = check_type($key, $value); };
2177             die "unable to parse value of '$key' - $@" if $@;
2178
2179             $cref->{$key} = $value;
2180
2181             if (!$snapname && is_valid_drivename($key)) {
2182                 my $drive = parse_drive($key, $value);
2183                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2184             }
2185         }
2186     };
2187
2188     &$cleanup_config($conf);
2189
2190     &$cleanup_config($conf->{pending}, 1);
2191
2192     foreach my $snapname (keys %{$conf->{snapshots}}) {
2193         die "internal error" if $snapname eq 'pending';
2194         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2195     }
2196
2197     # remove 'unusedX' settings if we re-add a volume
2198     foreach my $key (keys %$conf) {
2199         my $value = $conf->{$key};
2200         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2201             delete $conf->{$key};
2202         }
2203     }
2204
2205     my $generate_raw_config = sub {
2206         my ($conf, $pending) = @_;
2207
2208         my $raw = '';
2209
2210         # add description as comment to top of file
2211         if (defined(my $descr = $conf->{description})) {
2212             if ($descr) {
2213                 foreach my $cl (split(/\n/, $descr)) {
2214                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2215                 }
2216             } else {
2217                 $raw .= "#\n" if $pending;
2218             }
2219         }
2220
2221         foreach my $key (sort keys %$conf) {
2222             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2223             $raw .= "$key: $conf->{$key}\n";
2224         }
2225         return $raw;
2226     };
2227
2228     my $raw = &$generate_raw_config($conf);
2229
2230     if (scalar(keys %{$conf->{pending}})){
2231         $raw .= "\n[PENDING]\n";
2232         $raw .= &$generate_raw_config($conf->{pending}, 1);
2233     }
2234
2235     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2236         $raw .= "\n[$snapname]\n";
2237         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2238     }
2239
2240     return $raw;
2241 }
2242
2243 sub load_defaults {
2244
2245     my $res = {};
2246
2247     # we use static defaults from our JSON schema configuration
2248     foreach my $key (keys %$confdesc) {
2249         if (defined(my $default = $confdesc->{$key}->{default})) {
2250             $res->{$key} = $default;
2251         }
2252     }
2253
2254     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2255     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2256
2257     return $res;
2258 }
2259
2260 sub config_list {
2261     my $vmlist = PVE::Cluster::get_vmlist();
2262     my $res = {};
2263     return $res if !$vmlist || !$vmlist->{ids};
2264     my $ids = $vmlist->{ids};
2265
2266     foreach my $vmid (keys %$ids) {
2267         my $d = $ids->{$vmid};
2268         next if !$d->{node} || $d->{node} ne $nodename;
2269         next if !$d->{type} || $d->{type} ne 'qemu';
2270         $res->{$vmid}->{exists} = 1;
2271     }
2272     return $res;
2273 }
2274
2275 # test if VM uses local resources (to prevent migration)
2276 sub check_local_resources {
2277     my ($conf, $noerr) = @_;
2278
2279     my $loc_res = 0;
2280
2281     $loc_res = 1 if $conf->{hostusb}; # old syntax
2282     $loc_res = 1 if $conf->{hostpci}; # old syntax
2283
2284     foreach my $k (keys %$conf) {
2285         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2286         # sockets are safe: they will recreated be on the target side post-migrate
2287         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2288         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2289     }
2290
2291     die "VM uses local resources\n" if $loc_res && !$noerr;
2292
2293     return $loc_res;
2294 }
2295
2296 # check if used storages are available on all nodes (use by migrate)
2297 sub check_storage_availability {
2298     my ($storecfg, $conf, $node) = @_;
2299
2300     foreach_drive($conf, sub {
2301         my ($ds, $drive) = @_;
2302
2303         my $volid = $drive->{file};
2304         return if !$volid;
2305
2306         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2307         return if !$sid;
2308
2309         # check if storage is available on both nodes
2310         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2311         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2312    });
2313 }
2314
2315 # list nodes where all VM images are available (used by has_feature API)
2316 sub shared_nodes {
2317     my ($conf, $storecfg) = @_;
2318
2319     my $nodelist = PVE::Cluster::get_nodelist();
2320     my $nodehash = { map { $_ => 1 } @$nodelist };
2321     my $nodename = PVE::INotify::nodename();
2322
2323     foreach_drive($conf, sub {
2324         my ($ds, $drive) = @_;
2325
2326         my $volid = $drive->{file};
2327         return if !$volid;
2328
2329         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2330         if ($storeid) {
2331             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2332             if ($scfg->{disable}) {
2333                 $nodehash = {};
2334             } elsif (my $avail = $scfg->{nodes}) {
2335                 foreach my $node (keys %$nodehash) {
2336                     delete $nodehash->{$node} if !$avail->{$node};
2337                 }
2338             } elsif (!$scfg->{shared}) {
2339                 foreach my $node (keys %$nodehash) {
2340                     delete $nodehash->{$node} if $node ne $nodename
2341                 }
2342             }
2343         }
2344     });
2345
2346     return $nodehash
2347 }
2348
2349 sub check_cmdline {
2350     my ($pidfile, $pid) = @_;
2351
2352     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2353     if (defined($fh)) {
2354         my $line = <$fh>;
2355         $fh->close;
2356         return undef if !$line;
2357         my @param = split(/\0/, $line);
2358
2359         my $cmd = $param[0];
2360         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2361
2362         for (my $i = 0; $i < scalar (@param); $i++) {
2363             my $p = $param[$i];
2364             next if !$p;
2365             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2366                 my $p = $param[$i+1];
2367                 return 1 if $p && ($p eq $pidfile);
2368                 return undef;
2369             }
2370         }
2371     }
2372     return undef;
2373 }
2374
2375 sub check_running {
2376     my ($vmid, $nocheck, $node) = @_;
2377
2378     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2379
2380     die "unable to find configuration file for VM $vmid - no such machine\n"
2381         if !$nocheck && ! -f $filename;
2382
2383     my $pidfile = pidfile_name($vmid);
2384
2385     if (my $fd = IO::File->new("<$pidfile")) {
2386         my $st = stat($fd);
2387         my $line = <$fd>;
2388         close($fd);
2389
2390         my $mtime = $st->mtime;
2391         if ($mtime > time()) {
2392             warn "file '$filename' modified in future\n";
2393         }
2394
2395         if ($line =~ m/^(\d+)$/) {
2396             my $pid = $1;
2397             if (check_cmdline($pidfile, $pid)) {
2398                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2399                     return $pid;
2400                 }
2401             }
2402         }
2403     }
2404
2405     return undef;
2406 }
2407
2408 sub vzlist {
2409
2410     my $vzlist = config_list();
2411
2412     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2413
2414     while (defined(my $de = $fd->read)) {
2415         next if $de !~ m/^(\d+)\.pid$/;
2416         my $vmid = $1;
2417         next if !defined($vzlist->{$vmid});
2418         if (my $pid = check_running($vmid)) {
2419             $vzlist->{$vmid}->{pid} = $pid;
2420         }
2421     }
2422
2423     return $vzlist;
2424 }
2425
2426 sub disksize {
2427     my ($storecfg, $conf) = @_;
2428
2429     my $bootdisk = $conf->{bootdisk};
2430     return undef if !$bootdisk;
2431     return undef if !is_valid_drivename($bootdisk);
2432
2433     return undef if !$conf->{$bootdisk};
2434
2435     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2436     return undef if !defined($drive);
2437
2438     return undef if drive_is_cdrom($drive);
2439
2440     my $volid = $drive->{file};
2441     return undef if !$volid;
2442
2443     return $drive->{size};
2444 }
2445
2446 my $last_proc_pid_stat;
2447
2448 # get VM status information
2449 # This must be fast and should not block ($full == false)
2450 # We only query KVM using QMP if $full == true (this can be slow)
2451 sub vmstatus {
2452     my ($opt_vmid, $full) = @_;
2453
2454     my $res = {};
2455
2456     my $storecfg = PVE::Storage::config();
2457
2458     my $list = vzlist();
2459     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2460
2461     my $cpucount = $cpuinfo->{cpus} || 1;
2462
2463     foreach my $vmid (keys %$list) {
2464         next if $opt_vmid && ($vmid ne $opt_vmid);
2465
2466         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2467         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2468
2469         my $d = {};
2470         $d->{pid} = $list->{$vmid}->{pid};
2471
2472         # fixme: better status?
2473         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2474
2475         my $size = disksize($storecfg, $conf);
2476         if (defined($size)) {
2477             $d->{disk} = 0; # no info available
2478             $d->{maxdisk} = $size;
2479         } else {
2480             $d->{disk} = 0;
2481             $d->{maxdisk} = 0;
2482         }
2483
2484         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2485         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2486         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2487
2488         $d->{name} = $conf->{name} || "VM $vmid";
2489         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2490
2491         if ($conf->{balloon}) {
2492             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2493             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2494         }
2495
2496         $d->{uptime} = 0;
2497         $d->{cpu} = 0;
2498         $d->{mem} = 0;
2499
2500         $d->{netout} = 0;
2501         $d->{netin} = 0;
2502
2503         $d->{diskread} = 0;
2504         $d->{diskwrite} = 0;
2505
2506         $d->{template} = PVE::QemuConfig->is_template($conf);
2507
2508         $res->{$vmid} = $d;
2509     }
2510
2511     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2512     foreach my $dev (keys %$netdev) {
2513         next if $dev !~ m/^tap([1-9]\d*)i/;
2514         my $vmid = $1;
2515         my $d = $res->{$vmid};
2516         next if !$d;
2517
2518         $d->{netout} += $netdev->{$dev}->{receive};
2519         $d->{netin} += $netdev->{$dev}->{transmit};
2520
2521         if ($full) {
2522             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2523             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2524         }
2525
2526     }
2527
2528     my $ctime = gettimeofday;
2529
2530     foreach my $vmid (keys %$list) {
2531
2532         my $d = $res->{$vmid};
2533         my $pid = $d->{pid};
2534         next if !$pid;
2535
2536         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2537         next if !$pstat; # not running
2538
2539         my $used = $pstat->{utime} + $pstat->{stime};
2540
2541         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2542
2543         if ($pstat->{vsize}) {
2544             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2545         }
2546
2547         my $old = $last_proc_pid_stat->{$pid};
2548         if (!$old) {
2549             $last_proc_pid_stat->{$pid} = {
2550                 time => $ctime,
2551                 used => $used,
2552                 cpu => 0,
2553             };
2554             next;
2555         }
2556
2557         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2558
2559         if ($dtime > 1000) {
2560             my $dutime = $used -  $old->{used};
2561
2562             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2563             $last_proc_pid_stat->{$pid} = {
2564                 time => $ctime,
2565                 used => $used,
2566                 cpu => $d->{cpu},
2567             };
2568         } else {
2569             $d->{cpu} = $old->{cpu};
2570         }
2571     }
2572
2573     return $res if !$full;
2574
2575     my $qmpclient = PVE::QMPClient->new();
2576
2577     my $ballooncb = sub {
2578         my ($vmid, $resp) = @_;
2579
2580         my $info = $resp->{'return'};
2581         return if !$info->{max_mem};
2582
2583         my $d = $res->{$vmid};
2584
2585         # use memory assigned to VM
2586         $d->{maxmem} = $info->{max_mem};
2587         $d->{balloon} = $info->{actual};
2588
2589         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2590             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2591             $d->{freemem} = $info->{free_mem};
2592         }
2593
2594         $d->{ballooninfo} = $info;
2595     };
2596
2597     my $blockstatscb = sub {
2598         my ($vmid, $resp) = @_;
2599         my $data = $resp->{'return'} || [];
2600         my $totalrdbytes = 0;
2601         my $totalwrbytes = 0;
2602
2603         for my $blockstat (@$data) {
2604             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2605             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2606
2607             $blockstat->{device} =~ s/drive-//;
2608             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2609         }
2610         $res->{$vmid}->{diskread} = $totalrdbytes;
2611         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2612     };
2613
2614     my $statuscb = sub {
2615         my ($vmid, $resp) = @_;
2616
2617         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2618         # this fails if ballon driver is not loaded, so this must be
2619         # the last commnand (following command are aborted if this fails).
2620         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2621
2622         my $status = 'unknown';
2623         if (!defined($status = $resp->{'return'}->{status})) {
2624             warn "unable to get VM status\n";
2625             return;
2626         }
2627
2628         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2629     };
2630
2631     foreach my $vmid (keys %$list) {
2632         next if $opt_vmid && ($vmid ne $opt_vmid);
2633         next if !$res->{$vmid}->{pid}; # not running
2634         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2635     }
2636
2637     $qmpclient->queue_execute(undef, 1);
2638
2639     foreach my $vmid (keys %$list) {
2640         next if $opt_vmid && ($vmid ne $opt_vmid);
2641         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2642     }
2643
2644     return $res;
2645 }
2646
2647 sub foreach_dimm {
2648     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2649
2650     my $dimm_id = 0;
2651     my $current_size = 1024;
2652     my $dimm_size = 512;
2653     return if $current_size == $memory;
2654
2655     for (my $j = 0; $j < 8; $j++) {
2656         for (my $i = 0; $i < 32; $i++) {
2657             my $name = "dimm${dimm_id}";
2658             $dimm_id++;
2659             my $numanode = $i % $sockets;
2660             $current_size += $dimm_size;
2661             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2662             return  $current_size if $current_size >= $memory;
2663         }
2664         $dimm_size *= 2;
2665     }
2666 }
2667
2668 sub foreach_reverse_dimm {
2669     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2670
2671     my $dimm_id = 253;
2672     my $current_size = 4177920;
2673     my $dimm_size = 65536;
2674     return if $current_size == $memory;
2675
2676     for (my $j = 0; $j < 8; $j++) {
2677         for (my $i = 0; $i < 32; $i++) {
2678             my $name = "dimm${dimm_id}";
2679             $dimm_id--;
2680             my $numanode = $i % $sockets;
2681             $current_size -= $dimm_size;
2682             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2683             return  $current_size if $current_size <= $memory;
2684         }
2685         $dimm_size /= 2;
2686     }
2687 }
2688
2689 sub foreach_drive {
2690     my ($conf, $func) = @_;
2691
2692     foreach my $ds (valid_drive_names()) {
2693         next if !defined($conf->{$ds});
2694
2695         my $drive = parse_drive($ds, $conf->{$ds});
2696         next if !$drive;
2697
2698         &$func($ds, $drive);
2699     }
2700 }
2701
2702 sub foreach_volid {
2703     my ($conf, $func) = @_;
2704
2705     my $volhash = {};
2706
2707     my $test_volid = sub {
2708         my ($volid, $is_cdrom) = @_;
2709
2710         return if !$volid;
2711
2712         $volhash->{$volid} = $is_cdrom || 0;
2713     };
2714
2715     foreach_drive($conf, sub {
2716         my ($ds, $drive) = @_;
2717         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2718     });
2719
2720     foreach my $snapname (keys %{$conf->{snapshots}}) {
2721         my $snap = $conf->{snapshots}->{$snapname};
2722         &$test_volid($snap->{vmstate}, 0);
2723         foreach_drive($snap, sub {
2724             my ($ds, $drive) = @_;
2725             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2726         });
2727     }
2728
2729     foreach my $volid (keys %$volhash) {
2730         &$func($volid, $volhash->{$volid});
2731     }
2732 }
2733
2734 sub vga_conf_has_spice {
2735     my ($vga) = @_;
2736
2737     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2738
2739     return $1 || 1;
2740 }
2741
2742 sub config_to_command {
2743     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2744
2745     my $cmd = [];
2746     my $globalFlags = [];
2747     my $machineFlags = [];
2748     my $rtcFlags = [];
2749     my $cpuFlags = [];
2750     my $devices = [];
2751     my $pciaddr = '';
2752     my $bridges = {};
2753     my $kvmver = kvm_user_version();
2754     my $vernum = 0; # unknown
2755     my $ostype = $conf->{ostype};
2756     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2757         $vernum = $1*1000000+$2*1000;
2758     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2759         $vernum = $1*1000000+$2*1000+$3;
2760     }
2761
2762     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2763
2764     my $have_ovz = -f '/proc/vz/vestat';
2765
2766     my $q35 = machine_type_is_q35($conf);
2767     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2768     my $machine_type = $forcemachine || $conf->{machine};
2769     my $use_old_bios_files = undef;
2770     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2771
2772     my $cpuunits = defined($conf->{cpuunits}) ?
2773             $conf->{cpuunits} : $defaults->{cpuunits};
2774
2775     push @$cmd, '/usr/bin/systemd-run';
2776     push @$cmd, '--scope';
2777     push @$cmd, '--slice', "qemu";
2778     push @$cmd, '--unit', $vmid;
2779     push @$cmd, '--description', "'Proxmox VE VM $vmid'";
2780     # set KillMode=none, so that systemd don't kill those scopes
2781     # at shutdown (pve-manager service should stop the VMs instead)
2782     push @$cmd, '-p', "KillMode=none";
2783     push @$cmd, '-p', "CPUShares=$cpuunits";
2784     if ($conf->{cpulimit}) {
2785         my $cpulimit = int($conf->{cpulimit} * 100);
2786         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2787     }
2788
2789     push @$cmd, '/usr/bin/kvm';
2790
2791     push @$cmd, '-id', $vmid;
2792
2793     my $use_virtio = 0;
2794
2795     my $qmpsocket = qmp_socket($vmid);
2796     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2797     push @$cmd, '-mon', "chardev=qmp,mode=control";
2798
2799
2800     push @$cmd, '-pidfile' , pidfile_name($vmid);
2801
2802     push @$cmd, '-daemonize';
2803
2804     if ($conf->{smbios1}) {
2805         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2806     }
2807
2808     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2809         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2810         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2811         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2812         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2813         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2814         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2815     }
2816
2817     if ($q35) {
2818         # the q35 chipset support native usb2, so we enable usb controller
2819         # by default for this machine type
2820         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2821     } else {
2822         $pciaddr = print_pci_addr("piix3", $bridges);
2823         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2824
2825         my $use_usb2 = 0;
2826         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2827             next if !$conf->{"usb$i"};
2828             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2829             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2830             $use_usb2 = 1;
2831         }
2832         # include usb device config
2833         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2834     }
2835
2836     # add usb3 controller if needed
2837
2838     my $use_usb3 = 0;
2839     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2840         next if !$conf->{"usb$i"};
2841         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2842         next if !$d || !$d->{usb3};
2843         $use_usb3 = 1;
2844     }
2845
2846     $pciaddr = print_pci_addr("xhci", $bridges);
2847     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2848
2849     my $vga = $conf->{vga};
2850
2851     my $qxlnum = vga_conf_has_spice($vga);
2852     $vga = 'qxl' if $qxlnum;
2853
2854     if (!$vga) {
2855         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2856                                 $conf->{ostype} eq 'win7' ||
2857                                 $conf->{ostype} eq 'w2k8')) {
2858             $vga = 'std';
2859         } else {
2860             $vga = 'cirrus';
2861         }
2862     }
2863
2864     # enable absolute mouse coordinates (needed by vnc)
2865     my $tablet;
2866     if (defined($conf->{tablet})) {
2867         $tablet = $conf->{tablet};
2868     } else {
2869         $tablet = $defaults->{tablet};
2870         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2871         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2872     }
2873
2874     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2875
2876     my $kvm_off = 0;
2877     # host pci devices
2878     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2879         my $d = parse_hostpci($conf->{"hostpci$i"});
2880         next if !$d;
2881
2882         my $pcie = $d->{pcie};
2883         if($pcie){
2884             die "q35 machine model is not enabled" if !$q35;
2885             $pciaddr = print_pcie_addr("hostpci$i");
2886         }else{
2887             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2888         }
2889
2890         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2891         my $xvga = '';
2892         if ($d->{'x-vga'}) {
2893             $xvga = ',x-vga=on';
2894             $kvm_off = 1;
2895             $vga = 'none';
2896             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2897                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2898             }
2899             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2900                 $xvga = "";
2901             }
2902         }
2903         my $pcidevices = $d->{pciid};
2904         my $multifunction = 1 if @$pcidevices > 1;
2905
2906         my $j=0;
2907         foreach my $pcidevice (@$pcidevices) {
2908
2909             my $id = "hostpci$i";
2910             $id .= ".$j" if $multifunction;
2911             my $addr = $pciaddr;
2912             $addr .= ".$j" if $multifunction;
2913             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2914
2915             if($j == 0){
2916                 $devicestr .= "$rombar$xvga";
2917                 $devicestr .= ",multifunction=on" if $multifunction;
2918             }
2919
2920             push @$devices, '-device', $devicestr;
2921             $j++;
2922         }
2923     }
2924
2925     # usb devices
2926     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2927         next if !$conf->{"usb$i"};
2928         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2929         next if !$d;
2930
2931         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2932         my $usbbus = '';
2933         if (defined($d->{usb3}) && $d->{usb3}) {
2934             $usbbus = ',bus=xhci.0';
2935         }
2936
2937         if (defined($d->{host})) {
2938             $d = parse_usb_device($d->{host});
2939             if (defined($d->{vendorid}) && defined($d->{productid})) {
2940                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2941             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2942                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2943             } elsif (defined($d->{spice}) && $d->{spice}) {
2944                 # usb redir support for spice, currently no usb3
2945                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2946                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2947             }
2948         }
2949     }
2950
2951     # serial devices
2952     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2953         if (my $path = $conf->{"serial$i"}) {
2954             if ($path eq 'socket') {
2955                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2956                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2957                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2958             } else {
2959                 die "no such serial device\n" if ! -c $path;
2960                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2961                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2962             }
2963         }
2964     }
2965
2966     # parallel devices
2967     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2968         if (my $path = $conf->{"parallel$i"}) {
2969             die "no such parallel device\n" if ! -c $path;
2970             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2971             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2972             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2973         }
2974     }
2975
2976     my $vmname = $conf->{name} || "vm$vmid";
2977
2978     push @$cmd, '-name', $vmname;
2979
2980     my $sockets = 1;
2981     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2982     $sockets = $conf->{sockets} if  $conf->{sockets};
2983
2984     my $cores = $conf->{cores} || 1;
2985
2986     my $maxcpus = $sockets * $cores;
2987
2988     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2989
2990     my $allowed_vcpus = $cpuinfo->{cpus};
2991
2992     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2993         if ($allowed_vcpus < $maxcpus);
2994
2995     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2996
2997     push @$cmd, '-nodefaults';
2998
2999     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3000
3001     my $bootindex_hash = {};
3002     my $i = 1;
3003     foreach my $o (split(//, $bootorder)) {
3004         $bootindex_hash->{$o} = $i*100;
3005         $i++;
3006     }
3007
3008     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
3009
3010     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3011
3012     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3013
3014     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3015
3016     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3017         my $socket = vnc_socket($vmid);
3018         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3019     } else {
3020         push @$cmd, '-nographic';
3021     }
3022
3023     # time drift fix
3024     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3025
3026     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3027     my $useLocaltime = $conf->{localtime};
3028
3029     if ($ostype) {
3030         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3031
3032         if ($ostype =~ m/^w/) { # windows
3033             $useLocaltime = 1 if !defined($conf->{localtime});
3034
3035             # use time drift fix when acpi is enabled
3036             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3037                 $tdf = 1 if !defined($conf->{tdf});
3038             }
3039         }
3040
3041         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3042             $ostype eq 'wvista') {
3043             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3044             push @$cmd, '-no-hpet';
3045             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3046                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3047                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3048                 push @$cpuFlags , 'hv_time' if !$nokvm;
3049
3050             } else {
3051                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3052             }
3053         }
3054
3055         if ($ostype eq 'win7' || $ostype eq 'win8') {
3056             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3057         }
3058     }
3059
3060     push @$rtcFlags, 'driftfix=slew' if $tdf;
3061
3062     if ($nokvm) {
3063         push @$machineFlags, 'accel=tcg';
3064     } else {
3065         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3066     }
3067
3068     if ($machine_type) {
3069         push @$machineFlags, "type=${machine_type}";
3070     }
3071
3072     if ($conf->{startdate}) {
3073         push @$rtcFlags, "base=$conf->{startdate}";
3074     } elsif ($useLocaltime) {
3075         push @$rtcFlags, 'base=localtime';
3076     }
3077
3078     my $cpu = $nokvm ? "qemu64" : "kvm64";
3079     if (my $cputype = $conf->{cpu}) {
3080         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3081             or die "Cannot parse cpu description: $cputype\n";
3082         $cpu = $cpuconf->{cputype};
3083         $kvm_off = 1 if $cpuconf->{hidden};
3084     }
3085
3086     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3087
3088     push @$cpuFlags , '-x2apic'
3089         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3090
3091     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3092
3093     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3094
3095     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3096
3097         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3098         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3099     }
3100
3101     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3102
3103     push @$cpuFlags, 'kvm=off' if $kvm_off;
3104
3105     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3106         die "internal error"; # should not happen
3107
3108     push @$cpuFlags, "vendor=${cpu_vendor}"
3109         if $cpu_vendor ne 'default';
3110
3111     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3112
3113     push @$cmd, '-cpu', $cpu;
3114
3115     my $memory = $conf->{memory} || $defaults->{memory};
3116     my $static_memory = 0;
3117     my $dimm_memory = 0;
3118
3119     if ($hotplug_features->{memory}) {
3120         die "NUMA need to be enabled for memory hotplug\n" if !$conf->{numa};
3121         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3122         $static_memory = $STATICMEM;
3123         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3124         $dimm_memory = $memory - $static_memory;
3125         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3126
3127     } else {
3128
3129         $static_memory = $memory;
3130         push @$cmd, '-m', $static_memory;
3131     }
3132
3133     if ($conf->{numa}) {
3134
3135         my $numa_totalmemory = undef;
3136         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3137             next if !$conf->{"numa$i"};
3138             my $numa = parse_numa($conf->{"numa$i"});
3139             next if !$numa;
3140             # memory
3141             die "missing NUMA node$i memory value\n" if !$numa->{memory};
3142             my $numa_memory = $numa->{memory};
3143             $numa_totalmemory += $numa_memory;
3144             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3145
3146             # cpus
3147             my $cpulists = $numa->{cpus};
3148             die "missing NUMA node$i cpus\n" if !defined($cpulists);
3149             my $cpus = join(',', map {
3150                 my ($start, $end) = @$_;
3151                 defined($end) ? "$start-$end" : $start
3152             } @$cpulists);
3153
3154             # hostnodes
3155             my $hostnodelists = $numa->{hostnodes};
3156             if (defined($hostnodelists)) {
3157                 my $hostnodes;
3158                 foreach my $hostnoderange (@$hostnodelists) {
3159                     my ($start, $end) = @$hostnoderange;
3160                     $hostnodes .= ',' if $hostnodes;
3161                     $hostnodes .= $start;
3162                     $hostnodes .= "-$end" if defined($end);
3163                     $end //= $start;
3164                     for (my $i = $start; $i <= $end; ++$i ) {
3165                         die "host NUMA node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3166                     }
3167                 }
3168
3169                 # policy
3170                 my $policy = $numa->{policy};
3171                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3172                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3173             }
3174
3175             push @$cmd, '-object', $numa_object;
3176             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3177         }
3178
3179         die "total memory for NUMA nodes must be equal to vm static memory\n"
3180             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3181
3182         #if no custom tology, we split memory and cores across numa nodes
3183         if(!$numa_totalmemory) {
3184
3185             my $numa_memory = ($static_memory / $sockets) . "M";
3186
3187             for (my $i = 0; $i < $sockets; $i++)  {
3188
3189                 my $cpustart = ($cores * $i);
3190                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3191                 my $cpus = $cpustart;
3192                 $cpus .= "-$cpuend" if $cpuend;
3193
3194                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3195                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3196             }
3197         }
3198     }
3199
3200     if ($hotplug_features->{memory}) {
3201         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3202             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3203             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3204             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3205
3206             #if dimm_memory is not aligned to dimm map
3207             if($current_size > $memory) {
3208                  $conf->{memory} = $current_size;
3209                  PVE::QemuConfig->write_config($vmid, $conf);
3210             }
3211         });
3212     }
3213
3214     push @$cmd, '-S' if $conf->{freeze};
3215
3216     # set keyboard layout
3217     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3218     push @$cmd, '-k', $kb if $kb;
3219
3220     # enable sound
3221     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3222     #push @$cmd, '-soundhw', 'es1370';
3223     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3224
3225     if($conf->{agent}) {
3226         my $qgasocket = qmp_socket($vmid, 1);
3227         my $pciaddr = print_pci_addr("qga0", $bridges);
3228         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3229         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3230         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3231     }
3232
3233     my $spice_port;
3234
3235     if ($qxlnum) {
3236         if ($qxlnum > 1) {
3237             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3238                 for(my $i = 1; $i < $qxlnum; $i++){
3239                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3240                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3241                 }
3242             } else {
3243                 # assume other OS works like Linux
3244                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3245                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3246             }
3247         }
3248
3249         my $pciaddr = print_pci_addr("spice", $bridges);
3250
3251         my $nodename = PVE::INotify::nodename();
3252         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3253         $spice_port = PVE::Tools::next_spice_port($pfamily);
3254
3255         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3256
3257         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3258         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3259         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3260     }
3261
3262     # enable balloon by default, unless explicitly disabled
3263     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3264         $pciaddr = print_pci_addr("balloon0", $bridges);
3265         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3266     }
3267
3268     if ($conf->{watchdog}) {
3269         my $wdopts = parse_watchdog($conf->{watchdog});
3270         $pciaddr = print_pci_addr("watchdog", $bridges);
3271         my $watchdog = $wdopts->{model} || 'i6300esb';
3272         push @$devices, '-device', "$watchdog$pciaddr";
3273         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3274     }
3275
3276     my $vollist = [];
3277     my $scsicontroller = {};
3278     my $ahcicontroller = {};
3279     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3280
3281     # Add iscsi initiator name if available
3282     if (my $initiator = get_initiator_name()) {
3283         push @$devices, '-iscsi', "initiator-name=$initiator";
3284     }
3285
3286     foreach_drive($conf, sub {
3287         my ($ds, $drive) = @_;
3288
3289         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3290             push @$vollist, $drive->{file};
3291         }
3292
3293         $use_virtio = 1 if $ds =~ m/^virtio/;
3294
3295         if (drive_is_cdrom ($drive)) {
3296             if ($bootindex_hash->{d}) {
3297                 $drive->{bootindex} = $bootindex_hash->{d};
3298                 $bootindex_hash->{d} += 1;
3299             }
3300         } else {
3301             if ($bootindex_hash->{c}) {
3302                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3303                 $bootindex_hash->{c} += 1;
3304             }
3305         }
3306
3307         if($drive->{interface} eq 'virtio'){
3308            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3309         }
3310
3311         if ($drive->{interface} eq 'scsi') {
3312
3313             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3314
3315             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3316             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3317
3318             my $iothread = '';
3319             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3320                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3321                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3322             } elsif ($drive->{iothread}) {
3323                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3324             }
3325
3326             my $queues = '';
3327             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3328                 $queues = ",num_queues=$drive->{queues}";
3329             }
3330
3331             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3332             $scsicontroller->{$controller}=1;
3333         }
3334
3335         if ($drive->{interface} eq 'sata') {
3336            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3337            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3338            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3339            $ahcicontroller->{$controller}=1;
3340         }
3341
3342         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3343         push @$devices, '-drive',$drive_cmd;
3344         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3345     });
3346
3347     for (my $i = 0; $i < $MAX_NETS; $i++) {
3348          next if !$conf->{"net$i"};
3349          my $d = parse_net($conf->{"net$i"});
3350          next if !$d;
3351
3352          $use_virtio = 1 if $d->{model} eq 'virtio';
3353
3354          if ($bootindex_hash->{n}) {
3355             $d->{bootindex} = $bootindex_hash->{n};
3356             $bootindex_hash->{n} += 1;
3357          }
3358
3359          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3360          push @$devices, '-netdev', $netdevfull;
3361
3362          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3363          push @$devices, '-device', $netdevicefull;
3364     }
3365
3366     if (!$q35) {
3367         # add pci bridges
3368         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3369            $bridges->{1} = 1;
3370            $bridges->{2} = 1;
3371         }
3372
3373         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3374
3375         while (my ($k, $v) = each %$bridges) {
3376             $pciaddr = print_pci_addr("pci.$k");
3377             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3378         }
3379     }
3380
3381     # add custom args
3382     if ($conf->{args}) {
3383         my $aa = PVE::Tools::split_args($conf->{args});
3384         push @$cmd, @$aa;
3385     }
3386
3387     push @$cmd, @$devices;
3388     push @$cmd, '-rtc', join(',', @$rtcFlags)
3389         if scalar(@$rtcFlags);
3390     push @$cmd, '-machine', join(',', @$machineFlags)
3391         if scalar(@$machineFlags);
3392     push @$cmd, '-global', join(',', @$globalFlags)
3393         if scalar(@$globalFlags);
3394
3395     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3396 }
3397
3398 sub vnc_socket {
3399     my ($vmid) = @_;
3400     return "${var_run_tmpdir}/$vmid.vnc";
3401 }
3402
3403 sub spice_port {
3404     my ($vmid) = @_;
3405
3406     my $res = vm_mon_cmd($vmid, 'query-spice');
3407
3408     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3409 }
3410
3411 sub qmp_socket {
3412     my ($vmid, $qga) = @_;
3413     my $sockettype = $qga ? 'qga' : 'qmp';
3414     return "${var_run_tmpdir}/$vmid.$sockettype";
3415 }
3416
3417 sub pidfile_name {
3418     my ($vmid) = @_;
3419     return "${var_run_tmpdir}/$vmid.pid";
3420 }
3421
3422 sub vm_devices_list {
3423     my ($vmid) = @_;
3424
3425     my $res = vm_mon_cmd($vmid, 'query-pci');
3426     my $devices = {};
3427     foreach my $pcibus (@$res) {
3428         foreach my $device (@{$pcibus->{devices}}) {
3429             next if !$device->{'qdev_id'};
3430             if ($device->{'pci_bridge'}) {
3431                 $devices->{$device->{'qdev_id'}} = 1;
3432                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3433                     next if !$bridge_device->{'qdev_id'};
3434                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3435                     $devices->{$device->{'qdev_id'}}++;
3436                 }
3437             } else {
3438                 $devices->{$device->{'qdev_id'}} = 1;
3439             }
3440         }
3441     }
3442
3443     my $resblock = vm_mon_cmd($vmid, 'query-block');
3444     foreach my $block (@$resblock) {
3445         if($block->{device} =~ m/^drive-(\S+)/){
3446                 $devices->{$1} = 1;
3447         }
3448     }
3449
3450     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3451     foreach my $mice (@$resmice) {
3452         if ($mice->{name} eq 'QEMU HID Tablet') {
3453             $devices->{tablet} = 1;
3454             last;
3455         }
3456     }
3457
3458     return $devices;
3459 }
3460
3461 sub vm_deviceplug {
3462     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3463
3464     my $q35 = machine_type_is_q35($conf);
3465
3466     my $devices_list = vm_devices_list($vmid);
3467     return 1 if defined($devices_list->{$deviceid});
3468
3469     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3470
3471     if ($deviceid eq 'tablet') {
3472
3473         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3474
3475     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3476
3477         qemu_iothread_add($vmid, $deviceid, $device);
3478
3479         qemu_driveadd($storecfg, $vmid, $device);
3480         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3481
3482         qemu_deviceadd($vmid, $devicefull);
3483         eval { qemu_deviceaddverify($vmid, $deviceid); };
3484         if (my $err = $@) {
3485             eval { qemu_drivedel($vmid, $deviceid); };
3486             warn $@ if $@;
3487             die $err;
3488         }
3489
3490     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3491
3492
3493         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3494         my $pciaddr = print_pci_addr($deviceid);
3495         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3496
3497         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3498
3499         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3500             qemu_iothread_add($vmid, $deviceid, $device);
3501             $devicefull .= ",iothread=iothread-$deviceid";
3502         }
3503
3504         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3505             $devicefull .= ",num_queues=$device->{queues}";
3506         }
3507
3508         qemu_deviceadd($vmid, $devicefull);
3509         qemu_deviceaddverify($vmid, $deviceid);
3510
3511     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3512
3513         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3514         qemu_driveadd($storecfg, $vmid, $device);
3515
3516         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3517         eval { qemu_deviceadd($vmid, $devicefull); };
3518         if (my $err = $@) {
3519             eval { qemu_drivedel($vmid, $deviceid); };
3520             warn $@ if $@;
3521             die $err;
3522         }
3523
3524     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3525
3526         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3527
3528         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3529         my $use_old_bios_files = undef;
3530         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3531
3532         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3533         qemu_deviceadd($vmid, $netdevicefull);
3534         eval { qemu_deviceaddverify($vmid, $deviceid); };
3535         if (my $err = $@) {
3536             eval { qemu_netdevdel($vmid, $deviceid); };
3537             warn $@ if $@;
3538             die $err;
3539         }
3540
3541     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3542
3543         my $bridgeid = $2;
3544         my $pciaddr = print_pci_addr($deviceid);
3545         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3546
3547         qemu_deviceadd($vmid, $devicefull);
3548         qemu_deviceaddverify($vmid, $deviceid);
3549
3550     } else {
3551         die "can't hotplug device '$deviceid'\n";
3552     }
3553
3554     return 1;
3555 }
3556
3557 # fixme: this should raise exceptions on error!
3558 sub vm_deviceunplug {
3559     my ($vmid, $conf, $deviceid) = @_;
3560
3561     my $devices_list = vm_devices_list($vmid);
3562     return 1 if !defined($devices_list->{$deviceid});
3563
3564     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3565
3566     if ($deviceid eq 'tablet') {
3567
3568         qemu_devicedel($vmid, $deviceid);
3569
3570     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3571
3572         qemu_devicedel($vmid, $deviceid);
3573         qemu_devicedelverify($vmid, $deviceid);
3574         qemu_drivedel($vmid, $deviceid);
3575         qemu_iothread_del($conf, $vmid, $deviceid);
3576
3577     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3578
3579         qemu_devicedel($vmid, $deviceid);
3580         qemu_devicedelverify($vmid, $deviceid);
3581         qemu_iothread_del($conf, $vmid, $deviceid);
3582
3583     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3584
3585         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3586         my $device = parse_drive($deviceid, $conf->{$deviceid});
3587         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3588
3589         qemu_devicedel($vmid, $deviceid);
3590         qemu_drivedel($vmid, $deviceid);
3591         qemu_deletescsihw($conf, $vmid, $deviceid);
3592
3593     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3594
3595         qemu_devicedel($vmid, $deviceid);
3596         qemu_devicedelverify($vmid, $deviceid);
3597         qemu_netdevdel($vmid, $deviceid);
3598
3599     } else {
3600         die "can't unplug device '$deviceid'\n";
3601     }
3602
3603     return 1;
3604 }
3605
3606 sub qemu_deviceadd {
3607     my ($vmid, $devicefull) = @_;
3608
3609     $devicefull = "driver=".$devicefull;
3610     my %options =  split(/[=,]/, $devicefull);
3611
3612     vm_mon_cmd($vmid, "device_add" , %options);
3613 }
3614
3615 sub qemu_devicedel {
3616     my ($vmid, $deviceid) = @_;
3617
3618     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3619 }
3620
3621 sub qemu_iothread_add {
3622     my($vmid, $deviceid, $device) = @_;
3623
3624     if ($device->{iothread}) {
3625         my $iothreads = vm_iothreads_list($vmid);
3626         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3627     }
3628 }
3629
3630 sub qemu_iothread_del {
3631     my($conf, $vmid, $deviceid) = @_;
3632
3633     my $device = parse_drive($deviceid, $conf->{$deviceid});
3634     if ($device->{iothread}) {
3635         my $iothreads = vm_iothreads_list($vmid);
3636         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3637     }
3638 }
3639
3640 sub qemu_objectadd {
3641     my($vmid, $objectid, $qomtype) = @_;
3642
3643     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3644
3645     return 1;
3646 }
3647
3648 sub qemu_objectdel {
3649     my($vmid, $objectid) = @_;
3650
3651     vm_mon_cmd($vmid, "object-del", id => $objectid);
3652
3653     return 1;
3654 }
3655
3656 sub qemu_driveadd {
3657     my ($storecfg, $vmid, $device) = @_;
3658
3659     my $drive = print_drive_full($storecfg, $vmid, $device);
3660     $drive =~ s/\\/\\\\/g;
3661     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3662
3663     # If the command succeeds qemu prints: "OK"
3664     return 1 if $ret =~ m/OK/s;
3665
3666     die "adding drive failed: $ret\n";
3667 }
3668
3669 sub qemu_drivedel {
3670     my($vmid, $deviceid) = @_;
3671
3672     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3673     $ret =~ s/^\s+//;
3674
3675     return 1 if $ret eq "";
3676
3677     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3678     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3679
3680     die "deleting drive $deviceid failed : $ret\n";
3681 }
3682
3683 sub qemu_deviceaddverify {
3684     my ($vmid, $deviceid) = @_;
3685
3686     for (my $i = 0; $i <= 5; $i++) {
3687          my $devices_list = vm_devices_list($vmid);
3688          return 1 if defined($devices_list->{$deviceid});
3689          sleep 1;
3690     }
3691
3692     die "error on hotplug device '$deviceid'\n";
3693 }
3694
3695
3696 sub qemu_devicedelverify {
3697     my ($vmid, $deviceid) = @_;
3698
3699     # need to verify that the device is correctly removed as device_del
3700     # is async and empty return is not reliable
3701
3702     for (my $i = 0; $i <= 5; $i++) {
3703          my $devices_list = vm_devices_list($vmid);
3704          return 1 if !defined($devices_list->{$deviceid});
3705          sleep 1;
3706     }
3707
3708     die "error on hot-unplugging device '$deviceid'\n";
3709 }
3710
3711 sub qemu_findorcreatescsihw {
3712     my ($storecfg, $conf, $vmid, $device) = @_;
3713
3714     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3715
3716     my $scsihwid="$controller_prefix$controller";
3717     my $devices_list = vm_devices_list($vmid);
3718
3719     if(!defined($devices_list->{$scsihwid})) {
3720         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3721     }
3722
3723     return 1;
3724 }
3725
3726 sub qemu_deletescsihw {
3727     my ($conf, $vmid, $opt) = @_;
3728
3729     my $device = parse_drive($opt, $conf->{$opt});
3730
3731     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3732         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3733         return 1;
3734     }
3735
3736     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3737
3738     my $devices_list = vm_devices_list($vmid);
3739     foreach my $opt (keys %{$devices_list}) {
3740         if (PVE::QemuServer::is_valid_drivename($opt)) {
3741             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3742             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3743                 return 1;
3744             }
3745         }
3746     }
3747
3748     my $scsihwid="scsihw$controller";
3749
3750     vm_deviceunplug($vmid, $conf, $scsihwid);
3751
3752     return 1;
3753 }
3754
3755 sub qemu_add_pci_bridge {
3756     my ($storecfg, $conf, $vmid, $device) = @_;
3757
3758     my $bridges = {};
3759
3760     my $bridgeid;
3761
3762     print_pci_addr($device, $bridges);
3763
3764     while (my ($k, $v) = each %$bridges) {
3765         $bridgeid = $k;
3766     }
3767     return 1 if !defined($bridgeid) || $bridgeid < 1;
3768
3769     my $bridge = "pci.$bridgeid";
3770     my $devices_list = vm_devices_list($vmid);
3771
3772     if (!defined($devices_list->{$bridge})) {
3773         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3774     }
3775
3776     return 1;
3777 }
3778
3779 sub qemu_set_link_status {
3780     my ($vmid, $device, $up) = @_;
3781
3782     vm_mon_cmd($vmid, "set_link", name => $device,
3783                up => $up ? JSON::true : JSON::false);
3784 }
3785
3786 sub qemu_netdevadd {
3787     my ($vmid, $conf, $device, $deviceid) = @_;
3788
3789     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3790     my %options =  split(/[=,]/, $netdev);
3791
3792     vm_mon_cmd($vmid, "netdev_add",  %options);
3793     return 1;
3794 }
3795
3796 sub qemu_netdevdel {
3797     my ($vmid, $deviceid) = @_;
3798
3799     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3800 }
3801
3802 sub qemu_cpu_hotplug {
3803     my ($vmid, $conf, $vcpus) = @_;
3804
3805     my $sockets = 1;
3806     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3807     $sockets = $conf->{sockets} if  $conf->{sockets};
3808     my $cores = $conf->{cores} || 1;
3809     my $maxcpus = $sockets * $cores;
3810
3811     $vcpus = $maxcpus if !$vcpus;
3812
3813     die "you can't add more vcpus than maxcpus\n"
3814         if $vcpus > $maxcpus;
3815
3816     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3817     die "online cpu unplug is not yet possible\n"
3818         if $vcpus < $currentvcpus;
3819
3820     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3821     die "vcpus in running vm is different than configuration\n"
3822         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3823
3824     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3825         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3826     }
3827 }
3828
3829 sub qemu_memory_hotplug {
3830     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3831
3832     return $value if !check_running($vmid);
3833
3834     my $memory = $conf->{memory} || $defaults->{memory};
3835     $value = $defaults->{memory} if !$value;
3836     return $value if $value == $memory;
3837
3838     my $static_memory = $STATICMEM;
3839     my $dimm_memory = $memory - $static_memory;
3840
3841     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3842     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3843
3844
3845     my $sockets = 1;
3846     $sockets = $conf->{sockets} if $conf->{sockets};
3847
3848     if($value > $memory) {
3849
3850         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3851             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3852
3853                 return if $current_size <= $conf->{memory};
3854
3855                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3856                 if (my $err = $@) {
3857                     eval { qemu_objectdel($vmid, "mem-$name"); };
3858                     die $err;
3859                 }
3860
3861                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3862                 if (my $err = $@) {
3863                     eval { qemu_objectdel($vmid, "mem-$name"); };
3864                     die $err;
3865                 }
3866                 #update conf after each succesful module hotplug
3867                 $conf->{memory} = $current_size;
3868                 PVE::QemuConfig->write_config($vmid, $conf);
3869         });
3870
3871     } else {
3872
3873         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3874             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3875
3876                 return if $current_size >= $conf->{memory};
3877                 print "try to unplug memory dimm $name\n";
3878
3879                 my $retry = 0;
3880                 while (1) {
3881                     eval { qemu_devicedel($vmid, $name) };
3882                     sleep 3;
3883                     my $dimm_list = qemu_dimm_list($vmid);
3884                     last if !$dimm_list->{$name};
3885                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3886                     $retry++;
3887                 }
3888
3889                 #update conf after each succesful module unplug
3890                 $conf->{memory} = $current_size;
3891
3892                 eval { qemu_objectdel($vmid, "mem-$name"); };
3893                 PVE::QemuConfig->write_config($vmid, $conf);
3894         });
3895     }
3896 }
3897
3898 sub qemu_dimm_list {
3899     my ($vmid) = @_;
3900
3901     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3902     my $dimms = {};
3903
3904     foreach my $dimm (@$dimmarray) {
3905
3906         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3907         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3908         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3909         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3910         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3911     }
3912     return $dimms;
3913 }
3914
3915 sub qemu_block_set_io_throttle {
3916     my ($vmid, $deviceid,
3917         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3918         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3919
3920     return if !check_running($vmid) ;
3921
3922     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3923         bps => int($bps),
3924         bps_rd => int($bps_rd),
3925         bps_wr => int($bps_wr),
3926         iops => int($iops),
3927         iops_rd => int($iops_rd),
3928         iops_wr => int($iops_wr),
3929         bps_max => int($bps_max),
3930         bps_rd_max => int($bps_rd_max),
3931         bps_wr_max => int($bps_wr_max),
3932         iops_max => int($iops_max),
3933         iops_rd_max => int($iops_rd_max),
3934         iops_wr_max => int($iops_wr_max)
3935     );
3936
3937 }
3938
3939 # old code, only used to shutdown old VM after update
3940 sub __read_avail {
3941     my ($fh, $timeout) = @_;
3942
3943     my $sel = new IO::Select;
3944     $sel->add($fh);
3945
3946     my $res = '';
3947     my $buf;
3948
3949     my @ready;
3950     while (scalar (@ready = $sel->can_read($timeout))) {
3951         my $count;
3952         if ($count = $fh->sysread($buf, 8192)) {
3953             if ($buf =~ /^(.*)\(qemu\) $/s) {
3954                 $res .= $1;
3955                 last;
3956             } else {
3957                 $res .= $buf;
3958             }
3959         } else {
3960             if (!defined($count)) {
3961                 die "$!\n";
3962             }
3963             last;
3964         }
3965     }
3966
3967     die "monitor read timeout\n" if !scalar(@ready);
3968
3969     return $res;
3970 }
3971
3972 # old code, only used to shutdown old VM after update
3973 sub vm_monitor_command {
3974     my ($vmid, $cmdstr, $nocheck) = @_;
3975
3976     my $res;
3977
3978     eval {
3979         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3980
3981         my $sname = "${var_run_tmpdir}/$vmid.mon";
3982
3983         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3984             die "unable to connect to VM $vmid socket - $!\n";
3985
3986         my $timeout = 3;
3987
3988         # hack: migrate sometime blocks the monitor (when migrate_downtime
3989         # is set)
3990         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3991             $timeout = 60*60; # 1 hour
3992         }
3993
3994         # read banner;
3995         my $data = __read_avail($sock, $timeout);
3996
3997         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3998             die "got unexpected qemu monitor banner\n";
3999         }
4000
4001         my $sel = new IO::Select;
4002         $sel->add($sock);
4003
4004         if (!scalar(my @ready = $sel->can_write($timeout))) {
4005             die "monitor write error - timeout";
4006         }
4007
4008         my $fullcmd = "$cmdstr\r";
4009
4010         # syslog('info', "VM $vmid monitor command: $cmdstr");
4011
4012         my $b;
4013         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4014             die "monitor write error - $!";
4015         }
4016
4017         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4018
4019         $timeout = 20;
4020
4021         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4022             $timeout = 60*60; # 1 hour
4023         } elsif ($cmdstr =~ m/^(eject|change)/) {
4024             $timeout = 60; # note: cdrom mount command is slow
4025         }
4026         if ($res = __read_avail($sock, $timeout)) {
4027
4028             my @lines = split("\r?\n", $res);
4029
4030             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4031
4032             $res = join("\n", @lines);
4033             $res .= "\n";
4034         }
4035     };
4036
4037     my $err = $@;
4038
4039     if ($err) {
4040         syslog("err", "VM $vmid monitor command failed - $err");
4041         die $err;
4042     }
4043
4044     return $res;
4045 }
4046
4047 sub qemu_block_resize {
4048     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4049
4050     my $running = check_running($vmid);
4051
4052     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4053
4054     return if !$running;
4055
4056     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4057
4058 }
4059
4060 sub qemu_volume_snapshot {
4061     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4062
4063     my $running = check_running($vmid);
4064
4065     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4066         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4067     } else {
4068         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4069     }
4070 }
4071
4072 sub qemu_volume_snapshot_delete {
4073     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4074
4075     my $running = check_running($vmid);
4076
4077     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4078
4079     return if !$running;
4080
4081     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4082 }
4083
4084 sub set_migration_caps {
4085     my ($vmid) = @_;
4086
4087     my $cap_ref = [];
4088
4089     my $enabled_cap = {
4090         "auto-converge" => 1,
4091         "xbzrle" => 1,
4092         "x-rdma-pin-all" => 0,
4093         "zero-blocks" => 0,
4094         "compress" => 0
4095     };
4096
4097     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4098
4099     for my $supported_capability (@$supported_capabilities) {
4100         push @$cap_ref, {
4101             capability => $supported_capability->{capability},
4102             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4103         };
4104     }
4105
4106     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4107 }
4108
4109 my $fast_plug_option = {
4110     'lock' => 1,
4111     'name' => 1,
4112     'onboot' => 1,
4113     'shares' => 1,
4114     'startup' => 1,
4115     'description' => 1,
4116 };
4117
4118 # hotplug changes in [PENDING]
4119 # $selection hash can be used to only apply specified options, for
4120 # example: { cores => 1 } (only apply changed 'cores')
4121 # $errors ref is used to return error messages
4122 sub vmconfig_hotplug_pending {
4123     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4124
4125     my $defaults = load_defaults();
4126
4127     # commit values which do not have any impact on running VM first
4128     # Note: those option cannot raise errors, we we do not care about
4129     # $selection and always apply them.
4130
4131     my $add_error = sub {
4132         my ($opt, $msg) = @_;
4133         $errors->{$opt} = "hotplug problem - $msg";
4134     };
4135
4136     my $changes = 0;
4137     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4138         if ($fast_plug_option->{$opt}) {
4139             $conf->{$opt} = $conf->{pending}->{$opt};
4140             delete $conf->{pending}->{$opt};
4141             $changes = 1;
4142         }
4143     }
4144
4145     if ($changes) {
4146         PVE::QemuConfig->write_config($vmid, $conf);
4147         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4148     }
4149
4150     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4151
4152     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4153     while (my ($opt, $force) = each %$pending_delete_hash) {
4154         next if $selection && !$selection->{$opt};
4155         eval {
4156             if ($opt eq 'hotplug') {
4157                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4158             } elsif ($opt eq 'tablet') {
4159                 die "skip\n" if !$hotplug_features->{usb};
4160                 if ($defaults->{tablet}) {
4161                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4162                 } else {
4163                     vm_deviceunplug($vmid, $conf, $opt);
4164                 }
4165             } elsif ($opt eq 'vcpus') {
4166                 die "skip\n" if !$hotplug_features->{cpu};
4167                 qemu_cpu_hotplug($vmid, $conf, undef);
4168             } elsif ($opt eq 'balloon') {
4169                 # enable balloon device is not hotpluggable
4170                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4171             } elsif ($fast_plug_option->{$opt}) {
4172                 # do nothing
4173             } elsif ($opt =~ m/^net(\d+)$/) {
4174                 die "skip\n" if !$hotplug_features->{network};
4175                 vm_deviceunplug($vmid, $conf, $opt);
4176             } elsif (is_valid_drivename($opt)) {
4177                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4178                 vm_deviceunplug($vmid, $conf, $opt);
4179                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4180             } elsif ($opt =~ m/^memory$/) {
4181                 die "skip\n" if !$hotplug_features->{memory};
4182                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4183             } elsif ($opt eq 'cpuunits') {
4184                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4185             } elsif ($opt eq 'cpulimit') {
4186                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4187             } else {
4188                 die "skip\n";
4189             }
4190         };
4191         if (my $err = $@) {
4192             &$add_error($opt, $err) if $err ne "skip\n";
4193         } else {
4194             # save new config if hotplug was successful
4195             delete $conf->{$opt};
4196             vmconfig_undelete_pending_option($conf, $opt);
4197             PVE::QemuConfig->write_config($vmid, $conf);
4198             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4199         }
4200     }
4201
4202     foreach my $opt (keys %{$conf->{pending}}) {
4203         next if $selection && !$selection->{$opt};
4204         my $value = $conf->{pending}->{$opt};
4205         eval {
4206             if ($opt eq 'hotplug') {
4207                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4208             } elsif ($opt eq 'tablet') {
4209                 die "skip\n" if !$hotplug_features->{usb};
4210                 if ($value == 1) {
4211                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4212                 } elsif ($value == 0) {
4213                     vm_deviceunplug($vmid, $conf, $opt);
4214                 }
4215             } elsif ($opt eq 'vcpus') {
4216                 die "skip\n" if !$hotplug_features->{cpu};
4217                 qemu_cpu_hotplug($vmid, $conf, $value);
4218             } elsif ($opt eq 'balloon') {
4219                 # enable/disable balloning device is not hotpluggable
4220                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4221                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4222                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4223
4224                 # allow manual ballooning if shares is set to zero
4225                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4226                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4227                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4228                 }
4229             } elsif ($opt =~ m/^net(\d+)$/) {
4230                 # some changes can be done without hotplug
4231                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4232                                     $vmid, $opt, $value);
4233             } elsif (is_valid_drivename($opt)) {
4234                 # some changes can be done without hotplug
4235                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4236                                      $vmid, $opt, $value, 1);
4237             } elsif ($opt =~ m/^memory$/) { #dimms
4238                 die "skip\n" if !$hotplug_features->{memory};
4239                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4240             } elsif ($opt eq 'cpuunits') {
4241                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4242             } elsif ($opt eq 'cpulimit') {
4243                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4244                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4245             } else {
4246                 die "skip\n";  # skip non-hot-pluggable options
4247             }
4248         };
4249         if (my $err = $@) {
4250             &$add_error($opt, $err) if $err ne "skip\n";
4251         } else {
4252             # save new config if hotplug was successful
4253             $conf->{$opt} = $value;
4254             delete $conf->{pending}->{$opt};
4255             PVE::QemuConfig->write_config($vmid, $conf);
4256             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4257         }
4258     }
4259 }
4260
4261 sub try_deallocate_drive {
4262     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4263
4264     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4265         my $volid = $drive->{file};
4266         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4267             my $sid = PVE::Storage::parse_volume_id($volid);
4268             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4269
4270             # check if the disk is really unused
4271             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4272                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4273             PVE::Storage::vdisk_free($storecfg, $volid);
4274             return 1;
4275         } else {
4276             # If vm is not owner of this disk remove from config
4277             return 1;
4278         }
4279     }
4280
4281     return undef;
4282 }
4283
4284 sub vmconfig_delete_or_detach_drive {
4285     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4286
4287     my $drive = parse_drive($opt, $conf->{$opt});
4288
4289     my $rpcenv = PVE::RPCEnvironment::get();
4290     my $authuser = $rpcenv->get_user();
4291
4292     if ($force) {
4293         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4294         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4295     } else {
4296         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4297     }
4298 }
4299
4300 sub vmconfig_apply_pending {
4301     my ($vmid, $conf, $storecfg) = @_;
4302
4303     # cold plug
4304
4305     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4306     while (my ($opt, $force) = each %$pending_delete_hash) {
4307         die "internal error" if $opt =~ m/^unused/;
4308         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4309         if (!defined($conf->{$opt})) {
4310             vmconfig_undelete_pending_option($conf, $opt);
4311             PVE::QemuConfig->write_config($vmid, $conf);
4312         } elsif (is_valid_drivename($opt)) {
4313             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4314             vmconfig_undelete_pending_option($conf, $opt);
4315             delete $conf->{$opt};
4316             PVE::QemuConfig->write_config($vmid, $conf);
4317         } else {
4318             vmconfig_undelete_pending_option($conf, $opt);
4319             delete $conf->{$opt};
4320             PVE::QemuConfig->write_config($vmid, $conf);
4321         }
4322     }
4323
4324     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4325
4326     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4327         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4328
4329         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4330             # skip if nothing changed
4331         } elsif (is_valid_drivename($opt)) {
4332             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4333                 if defined($conf->{$opt});
4334             $conf->{$opt} = $conf->{pending}->{$opt};
4335         } else {
4336             $conf->{$opt} = $conf->{pending}->{$opt};
4337         }
4338
4339         delete $conf->{pending}->{$opt};
4340         PVE::QemuConfig->write_config($vmid, $conf);
4341     }
4342 }
4343
4344 my $safe_num_ne = sub {
4345     my ($a, $b) = @_;
4346
4347     return 0 if !defined($a) && !defined($b);
4348     return 1 if !defined($a);
4349     return 1 if !defined($b);
4350
4351     return $a != $b;
4352 };
4353
4354 my $safe_string_ne = sub {
4355     my ($a, $b) = @_;
4356
4357     return 0 if !defined($a) && !defined($b);
4358     return 1 if !defined($a);
4359     return 1 if !defined($b);
4360
4361     return $a ne $b;
4362 };
4363
4364 sub vmconfig_update_net {
4365     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4366
4367     my $newnet = parse_net($value);
4368
4369     if ($conf->{$opt}) {
4370         my $oldnet = parse_net($conf->{$opt});
4371
4372         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4373             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4374             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4375             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4376
4377             # for non online change, we try to hot-unplug
4378             die "skip\n" if !$hotplug;
4379             vm_deviceunplug($vmid, $conf, $opt);
4380         } else {
4381
4382             die "internal error" if $opt !~ m/net(\d+)/;
4383             my $iface = "tap${vmid}i$1";
4384
4385             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4386                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4387                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4388                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4389                 PVE::Network::tap_unplug($iface);
4390                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4391             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4392                 # Rate can be applied on its own but any change above needs to
4393                 # include the rate in tap_plug since OVS resets everything.
4394                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4395             }
4396
4397             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4398                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4399             }
4400
4401             return 1;
4402         }
4403     }
4404
4405     if ($hotplug) {
4406         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4407     } else {
4408         die "skip\n";
4409     }
4410 }
4411
4412 sub vmconfig_update_disk {
4413     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4414
4415     # fixme: do we need force?
4416
4417     my $drive = parse_drive($opt, $value);
4418
4419     if ($conf->{$opt}) {
4420
4421         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4422
4423             my $media = $drive->{media} || 'disk';
4424             my $oldmedia = $old_drive->{media} || 'disk';
4425             die "unable to change media type\n" if $media ne $oldmedia;
4426
4427             if (!drive_is_cdrom($old_drive)) {
4428
4429                 if ($drive->{file} ne $old_drive->{file}) {
4430
4431                     die "skip\n" if !$hotplug;
4432
4433                     # unplug and register as unused
4434                     vm_deviceunplug($vmid, $conf, $opt);
4435                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4436
4437                 } else {
4438                     # update existing disk
4439
4440                     # skip non hotpluggable value
4441                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4442                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4443                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4444                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4445                         die "skip\n";
4446                     }
4447
4448                     # apply throttle
4449                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4450                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4451                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4452                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4453                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4454                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4455                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4456                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4457                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4458                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4459                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4460                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4461
4462                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4463                                                    ($drive->{mbps} || 0)*1024*1024,
4464                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4465                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4466                                                    $drive->{iops} || 0,
4467                                                    $drive->{iops_rd} || 0,
4468                                                    $drive->{iops_wr} || 0,
4469                                                    ($drive->{mbps_max} || 0)*1024*1024,
4470                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4471                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4472                                                    $drive->{iops_max} || 0,
4473                                                    $drive->{iops_rd_max} || 0,
4474                                                    $drive->{iops_wr_max} || 0);
4475
4476                     }
4477
4478                     return 1;
4479                 }
4480
4481             } else { # cdrom
4482
4483                 if ($drive->{file} eq 'none') {
4484                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4485                 } else {
4486                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4487                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4488                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4489                 }
4490
4491                 return 1;
4492             }
4493         }
4494     }
4495
4496     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4497     # hotplug new disks
4498     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4499     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4500 }
4501
4502 sub vm_start {
4503     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4504         $forcemachine, $spice_ticket) = @_;
4505
4506     PVE::QemuConfig->lock_config($vmid, sub {
4507         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4508
4509         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4510
4511         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4512
4513         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4514
4515         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4516             vmconfig_apply_pending($vmid, $conf, $storecfg);
4517             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4518         }
4519
4520         my $defaults = load_defaults();
4521
4522         # set environment variable useful inside network script
4523         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4524
4525         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4526
4527         my $migrate_port = 0;
4528         my $migrate_uri;
4529         if ($statefile) {
4530             if ($statefile eq 'tcp') {
4531                 my $localip = "localhost";
4532                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4533                 my $nodename = PVE::INotify::nodename();
4534                 if ($datacenterconf->{migration_unsecure}) {
4535                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4536                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4537                 }
4538                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4539                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4540                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4541                 push @$cmd, '-incoming', $migrate_uri;
4542                 push @$cmd, '-S';
4543             } else {
4544                 push @$cmd, '-loadstate', $statefile;
4545             }
4546         } elsif ($paused) {
4547             push @$cmd, '-S';
4548         }
4549
4550         # host pci devices
4551         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4552           my $d = parse_hostpci($conf->{"hostpci$i"});
4553           next if !$d;
4554           my $pcidevices = $d->{pciid};
4555           foreach my $pcidevice (@$pcidevices) {
4556                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4557
4558                 my $info = pci_device_info("0000:$pciid");
4559                 die "IOMMU not present\n" if !check_iommu_support();
4560                 die "no pci device info for device '$pciid'\n" if !$info;
4561                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4562                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4563           }
4564         }
4565
4566         PVE::Storage::activate_volumes($storecfg, $vollist);
4567
4568         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4569             my $cmd = [];
4570             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4571             eval  { run_command($cmd); };
4572         }
4573
4574         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4575                     umask => 0077); };
4576
4577         if (my $err = $@) {
4578             # deactivate volumes if start fails
4579             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4580             die "start failed: $err";
4581         }
4582
4583         print "migration listens on $migrate_uri\n" if $migrate_uri;
4584
4585         if ($statefile && $statefile ne 'tcp')  {
4586             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4587             warn $@ if $@;
4588         }
4589
4590         if ($migratedfrom) {
4591
4592             eval {
4593                 set_migration_caps($vmid);
4594             };
4595             warn $@ if $@;
4596
4597             if ($spice_port) {
4598                 print "spice listens on port $spice_port\n";
4599                 if ($spice_ticket) {
4600                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4601                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4602                 }
4603             }
4604
4605         } else {
4606
4607             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4608                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4609                     if $conf->{balloon};
4610             }
4611
4612             foreach my $opt (keys %$conf) {
4613                 next if $opt !~  m/^net\d+$/;
4614                 my $nicconf = parse_net($conf->{$opt});
4615                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4616             }
4617         }
4618
4619         vm_mon_cmd_nocheck($vmid, 'qom-set',
4620                     path => "machine/peripheral/balloon0",
4621                     property => "guest-stats-polling-interval",
4622                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4623
4624     });
4625 }
4626
4627 sub vm_mon_cmd {
4628     my ($vmid, $execute, %params) = @_;
4629
4630     my $cmd = { execute => $execute, arguments => \%params };
4631     vm_qmp_command($vmid, $cmd);
4632 }
4633
4634 sub vm_mon_cmd_nocheck {
4635     my ($vmid, $execute, %params) = @_;
4636
4637     my $cmd = { execute => $execute, arguments => \%params };
4638     vm_qmp_command($vmid, $cmd, 1);
4639 }
4640
4641 sub vm_qmp_command {
4642     my ($vmid, $cmd, $nocheck) = @_;
4643
4644     my $res;
4645
4646     my $timeout;
4647     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4648         $timeout = $cmd->{arguments}->{timeout};
4649         delete $cmd->{arguments}->{timeout};
4650     }
4651
4652     eval {
4653         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4654         my $sname = qmp_socket($vmid);
4655         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4656             my $qmpclient = PVE::QMPClient->new();
4657
4658             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4659         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4660             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4661                 if scalar(%{$cmd->{arguments}});
4662             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4663         } else {
4664             die "unable to open monitor socket\n";
4665         }
4666     };
4667     if (my $err = $@) {
4668         syslog("err", "VM $vmid qmp command failed - $err");
4669         die $err;
4670     }
4671
4672     return $res;
4673 }
4674
4675 sub vm_human_monitor_command {
4676     my ($vmid, $cmdline) = @_;
4677
4678     my $res;
4679
4680     my $cmd = {
4681         execute => 'human-monitor-command',
4682         arguments => { 'command-line' => $cmdline},
4683     };
4684
4685     return vm_qmp_command($vmid, $cmd);
4686 }
4687
4688 sub vm_commandline {
4689     my ($storecfg, $vmid) = @_;
4690
4691     my $conf = PVE::QemuConfig->load_config($vmid);
4692
4693     my $defaults = load_defaults();
4694
4695     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4696
4697     return join(' ', @$cmd);
4698 }
4699
4700 sub vm_reset {
4701     my ($vmid, $skiplock) = @_;
4702
4703     PVE::QemuConfig->lock_config($vmid, sub {
4704
4705         my $conf = PVE::QemuConfig->load_config($vmid);
4706
4707         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4708
4709         vm_mon_cmd($vmid, "system_reset");
4710     });
4711 }
4712
4713 sub get_vm_volumes {
4714     my ($conf) = @_;
4715
4716     my $vollist = [];
4717     foreach_volid($conf, sub {
4718         my ($volid, $is_cdrom) = @_;
4719
4720         return if $volid =~ m|^/|;
4721
4722         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4723         return if !$sid;
4724
4725         push @$vollist, $volid;
4726     });
4727
4728     return $vollist;
4729 }
4730
4731 sub vm_stop_cleanup {
4732     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4733
4734     eval {
4735
4736         if (!$keepActive) {
4737             my $vollist = get_vm_volumes($conf);
4738             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4739         }
4740
4741         foreach my $ext (qw(mon qmp pid vnc qga)) {
4742             unlink "/var/run/qemu-server/${vmid}.$ext";
4743         }
4744
4745         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4746     };
4747     warn $@ if $@; # avoid errors - just warn
4748 }
4749
4750 # Note: use $nockeck to skip tests if VM configuration file exists.
4751 # We need that when migration VMs to other nodes (files already moved)
4752 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4753 sub vm_stop {
4754     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4755
4756     $force = 1 if !defined($force) && !$shutdown;
4757
4758     if ($migratedfrom){
4759         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4760         kill 15, $pid if $pid;
4761         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4762         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4763         return;
4764     }
4765
4766     PVE::QemuConfig->lock_config($vmid, sub {
4767
4768         my $pid = check_running($vmid, $nocheck);
4769         return if !$pid;
4770
4771         my $conf;
4772         if (!$nocheck) {
4773             $conf = PVE::QemuConfig->load_config($vmid);
4774             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4775             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4776                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4777                 $timeout = $opts->{down} if $opts->{down};
4778             }
4779         }
4780
4781         $timeout = 60 if !defined($timeout);
4782
4783         eval {
4784             if ($shutdown) {
4785                 if (defined($conf) && $conf->{agent}) {
4786                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4787                 } else {
4788                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4789                 }
4790             } else {
4791                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4792             }
4793         };
4794         my $err = $@;
4795
4796         if (!$err) {
4797             my $count = 0;
4798             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4799                 $count++;
4800                 sleep 1;
4801             }
4802
4803             if ($count >= $timeout) {
4804                 if ($force) {
4805                     warn "VM still running - terminating now with SIGTERM\n";
4806                     kill 15, $pid;
4807                 } else {
4808                     die "VM quit/powerdown failed - got timeout\n";
4809                 }
4810             } else {
4811                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4812                 return;
4813             }
4814         } else {
4815             if ($force) {
4816                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4817                 kill 15, $pid;
4818             } else {
4819                 die "VM quit/powerdown failed\n";
4820             }
4821         }
4822
4823         # wait again
4824         $timeout = 10;
4825
4826         my $count = 0;
4827         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4828             $count++;
4829             sleep 1;
4830         }
4831
4832         if ($count >= $timeout) {
4833             warn "VM still running - terminating now with SIGKILL\n";
4834             kill 9, $pid;
4835             sleep 1;
4836         }
4837
4838         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4839    });
4840 }
4841
4842 sub vm_suspend {
4843     my ($vmid, $skiplock) = @_;
4844
4845     PVE::QemuConfig->lock_config($vmid, sub {
4846
4847         my $conf = PVE::QemuConfig->load_config($vmid);
4848
4849         PVE::QemuConfig->check_lock($conf)
4850             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4851
4852         vm_mon_cmd($vmid, "stop");
4853     });
4854 }
4855
4856 sub vm_resume {
4857     my ($vmid, $skiplock, $nocheck) = @_;
4858
4859     PVE::QemuConfig->lock_config($vmid, sub {
4860
4861         if (!$nocheck) {
4862
4863             my $conf = PVE::QemuConfig->load_config($vmid);
4864
4865             PVE::QemuConfig->check_lock($conf)
4866                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4867
4868             vm_mon_cmd($vmid, "cont");
4869
4870         } else {
4871             vm_mon_cmd_nocheck($vmid, "cont");
4872         }
4873     });
4874 }
4875
4876 sub vm_sendkey {
4877     my ($vmid, $skiplock, $key) = @_;
4878
4879     PVE::QemuConfig->lock_config($vmid, sub {
4880
4881         my $conf = PVE::QemuConfig->load_config($vmid);
4882
4883         # there is no qmp command, so we use the human monitor command
4884         vm_human_monitor_command($vmid, "sendkey $key");
4885     });
4886 }
4887
4888 sub vm_destroy {
4889     my ($storecfg, $vmid, $skiplock) = @_;
4890
4891     PVE::QemuConfig->lock_config($vmid, sub {
4892
4893         my $conf = PVE::QemuConfig->load_config($vmid);
4894
4895         if (!check_running($vmid)) {
4896             destroy_vm($storecfg, $vmid, undef, $skiplock);
4897         } else {
4898             die "VM $vmid is running - destroy failed\n";
4899         }
4900     });
4901 }
4902
4903 # pci helpers
4904
4905 sub file_write {
4906     my ($filename, $buf) = @_;
4907
4908     my $fh = IO::File->new($filename, "w");
4909     return undef if !$fh;
4910
4911     my $res = print $fh $buf;
4912
4913     $fh->close();
4914
4915     return $res;
4916 }
4917
4918 sub pci_device_info {
4919     my ($name) = @_;
4920
4921     my $res;
4922
4923     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4924     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4925
4926     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4927     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4928
4929     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4930     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4931
4932     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4933     return undef if !defined($product) || $product !~ s/^0x//;
4934
4935     $res = {
4936         name => $name,
4937         vendor => $vendor,
4938         product => $product,
4939         domain => $domain,
4940         bus => $bus,
4941         slot => $slot,
4942         func => $func,
4943         irq => $irq,
4944         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4945     };
4946
4947     return $res;
4948 }
4949
4950 sub pci_dev_reset {
4951     my ($dev) = @_;
4952
4953     my $name = $dev->{name};
4954
4955     my $fn = "$pcisysfs/devices/$name/reset";
4956
4957     return file_write($fn, "1");
4958 }
4959
4960 sub pci_dev_bind_to_vfio {
4961     my ($dev) = @_;
4962
4963     my $name = $dev->{name};
4964
4965     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4966
4967     if (!-d $vfio_basedir) {
4968         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4969     }
4970     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4971
4972     my $testdir = "$vfio_basedir/$name";
4973     return 1 if -d $testdir;
4974
4975     my $data = "$dev->{vendor} $dev->{product}";
4976     return undef if !file_write("$vfio_basedir/new_id", $data);
4977
4978     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4979     if (!file_write($fn, $name)) {
4980         return undef if -f $fn;
4981     }
4982
4983     $fn = "$vfio_basedir/bind";
4984     if (! -d $testdir) {
4985         return undef if !file_write($fn, $name);
4986     }
4987
4988     return -d $testdir;
4989 }
4990
4991 sub pci_dev_group_bind_to_vfio {
4992     my ($pciid) = @_;
4993
4994     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4995
4996     if (!-d $vfio_basedir) {
4997         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4998     }
4999     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5000
5001     # get IOMMU group devices
5002     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5003       my @devs = grep /^0000:/, readdir($D);
5004     closedir($D);
5005
5006     foreach my $pciid (@devs) {
5007         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5008
5009         # pci bridges, switches or root ports are not supported
5010         # they have a pci_bus subdirectory so skip them
5011         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5012
5013         my $info = pci_device_info($1);
5014         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5015     }
5016
5017     return 1;
5018 }
5019
5020 sub print_pci_addr {
5021     my ($id, $bridges) = @_;
5022
5023     my $res = '';
5024     my $devices = {
5025         piix3 => { bus => 0, addr => 1 },
5026         #addr2 : first videocard
5027         balloon0 => { bus => 0, addr => 3 },
5028         watchdog => { bus => 0, addr => 4 },
5029         scsihw0 => { bus => 0, addr => 5 },
5030         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
5031         scsihw1 => { bus => 0, addr => 6 },
5032         ahci0 => { bus => 0, addr => 7 },
5033         qga0 => { bus => 0, addr => 8 },
5034         spice => { bus => 0, addr => 9 },
5035         virtio0 => { bus => 0, addr => 10 },
5036         virtio1 => { bus => 0, addr => 11 },
5037         virtio2 => { bus => 0, addr => 12 },
5038         virtio3 => { bus => 0, addr => 13 },
5039         virtio4 => { bus => 0, addr => 14 },
5040         virtio5 => { bus => 0, addr => 15 },
5041         hostpci0 => { bus => 0, addr => 16 },
5042         hostpci1 => { bus => 0, addr => 17 },
5043         net0 => { bus => 0, addr => 18 },
5044         net1 => { bus => 0, addr => 19 },
5045         net2 => { bus => 0, addr => 20 },
5046         net3 => { bus => 0, addr => 21 },
5047         net4 => { bus => 0, addr => 22 },
5048         net5 => { bus => 0, addr => 23 },
5049         vga1 => { bus => 0, addr => 24 },
5050         vga2 => { bus => 0, addr => 25 },
5051         vga3 => { bus => 0, addr => 26 },
5052         hostpci2 => { bus => 0, addr => 27 },
5053         hostpci3 => { bus => 0, addr => 28 },
5054         #addr29 : usb-host (pve-usb.cfg)
5055         'pci.1' => { bus => 0, addr => 30 },
5056         'pci.2' => { bus => 0, addr => 31 },
5057         'net6' => { bus => 1, addr => 1 },
5058         'net7' => { bus => 1, addr => 2 },
5059         'net8' => { bus => 1, addr => 3 },
5060         'net9' => { bus => 1, addr => 4 },
5061         'net10' => { bus => 1, addr => 5 },
5062         'net11' => { bus => 1, addr => 6 },
5063         'net12' => { bus => 1, addr => 7 },
5064         'net13' => { bus => 1, addr => 8 },
5065         'net14' => { bus => 1, addr => 9 },
5066         'net15' => { bus => 1, addr => 10 },
5067         'net16' => { bus => 1, addr => 11 },
5068         'net17' => { bus => 1, addr => 12 },
5069         'net18' => { bus => 1, addr => 13 },
5070         'net19' => { bus => 1, addr => 14 },
5071         'net20' => { bus => 1, addr => 15 },
5072         'net21' => { bus => 1, addr => 16 },
5073         'net22' => { bus => 1, addr => 17 },
5074         'net23' => { bus => 1, addr => 18 },
5075         'net24' => { bus => 1, addr => 19 },
5076         'net25' => { bus => 1, addr => 20 },
5077         'net26' => { bus => 1, addr => 21 },
5078         'net27' => { bus => 1, addr => 22 },
5079         'net28' => { bus => 1, addr => 23 },
5080         'net29' => { bus => 1, addr => 24 },
5081         'net30' => { bus => 1, addr => 25 },
5082         'net31' => { bus => 1, addr => 26 },
5083         'xhci' => { bus => 1, addr => 27 },
5084         'virtio6' => { bus => 2, addr => 1 },
5085         'virtio7' => { bus => 2, addr => 2 },
5086         'virtio8' => { bus => 2, addr => 3 },
5087         'virtio9' => { bus => 2, addr => 4 },
5088         'virtio10' => { bus => 2, addr => 5 },
5089         'virtio11' => { bus => 2, addr => 6 },
5090         'virtio12' => { bus => 2, addr => 7 },
5091         'virtio13' => { bus => 2, addr => 8 },
5092         'virtio14' => { bus => 2, addr => 9 },
5093         'virtio15' => { bus => 2, addr => 10 },
5094         'virtioscsi0' => { bus => 3, addr => 1 },
5095         'virtioscsi1' => { bus => 3, addr => 2 },
5096         'virtioscsi2' => { bus => 3, addr => 3 },
5097         'virtioscsi3' => { bus => 3, addr => 4 },
5098         'virtioscsi4' => { bus => 3, addr => 5 },
5099         'virtioscsi5' => { bus => 3, addr => 6 },
5100         'virtioscsi6' => { bus => 3, addr => 7 },
5101         'virtioscsi7' => { bus => 3, addr => 8 },
5102         'virtioscsi8' => { bus => 3, addr => 9 },
5103         'virtioscsi9' => { bus => 3, addr => 10 },
5104         'virtioscsi10' => { bus => 3, addr => 11 },
5105         'virtioscsi11' => { bus => 3, addr => 12 },
5106         'virtioscsi12' => { bus => 3, addr => 13 },
5107         'virtioscsi13' => { bus => 3, addr => 14 },
5108         'virtioscsi14' => { bus => 3, addr => 15 },
5109         'virtioscsi15' => { bus => 3, addr => 16 },
5110         'virtioscsi16' => { bus => 3, addr => 17 },
5111         'virtioscsi17' => { bus => 3, addr => 18 },
5112         'virtioscsi18' => { bus => 3, addr => 19 },
5113         'virtioscsi19' => { bus => 3, addr => 20 },
5114         'virtioscsi20' => { bus => 3, addr => 21 },
5115         'virtioscsi21' => { bus => 3, addr => 22 },
5116         'virtioscsi22' => { bus => 3, addr => 23 },
5117         'virtioscsi23' => { bus => 3, addr => 24 },
5118         'virtioscsi24' => { bus => 3, addr => 25 },
5119         'virtioscsi25' => { bus => 3, addr => 26 },
5120         'virtioscsi26' => { bus => 3, addr => 27 },
5121         'virtioscsi27' => { bus => 3, addr => 28 },
5122         'virtioscsi28' => { bus => 3, addr => 29 },
5123         'virtioscsi29' => { bus => 3, addr => 30 },
5124         'virtioscsi30' => { bus => 3, addr => 31 },
5125
5126     };
5127
5128     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5129            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5130            my $bus = $devices->{$id}->{bus};
5131            $res = ",bus=pci.$bus,addr=$addr";
5132            $bridges->{$bus} = 1 if $bridges;
5133     }
5134     return $res;
5135
5136 }
5137
5138 sub print_pcie_addr {
5139     my ($id) = @_;
5140
5141     my $res = '';
5142     my $devices = {
5143         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5144         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5145         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5146         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5147     };
5148
5149     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5150            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5151            my $bus = $devices->{$id}->{bus};
5152            $res = ",bus=$bus,addr=$addr";
5153     }
5154     return $res;
5155
5156 }
5157
5158 # vzdump restore implementaion
5159
5160 sub tar_archive_read_firstfile {
5161     my $archive = shift;
5162
5163     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5164
5165     # try to detect archive type first
5166     my $pid = open (TMP, "tar tf '$archive'|") ||
5167         die "unable to open file '$archive'\n";
5168     my $firstfile = <TMP>;
5169     kill 15, $pid;
5170     close TMP;
5171
5172     die "ERROR: archive contaions no data\n" if !$firstfile;
5173     chomp $firstfile;
5174
5175     return $firstfile;
5176 }
5177
5178 sub tar_restore_cleanup {
5179     my ($storecfg, $statfile) = @_;
5180
5181     print STDERR "starting cleanup\n";
5182
5183     if (my $fd = IO::File->new($statfile, "r")) {
5184         while (defined(my $line = <$fd>)) {
5185             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5186                 my $volid = $2;
5187                 eval {
5188                     if ($volid =~ m|^/|) {
5189                         unlink $volid || die 'unlink failed\n';
5190                     } else {
5191                         PVE::Storage::vdisk_free($storecfg, $volid);
5192                     }
5193                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5194                 };
5195                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5196             } else {
5197                 print STDERR "unable to parse line in statfile - $line";
5198             }
5199         }
5200         $fd->close();
5201     }
5202 }
5203
5204 sub restore_archive {
5205     my ($archive, $vmid, $user, $opts) = @_;
5206
5207     my $format = $opts->{format};
5208     my $comp;
5209
5210     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5211         $format = 'tar' if !$format;
5212         $comp = 'gzip';
5213     } elsif ($archive =~ m/\.tar$/) {
5214         $format = 'tar' if !$format;
5215     } elsif ($archive =~ m/.tar.lzo$/) {
5216         $format = 'tar' if !$format;
5217         $comp = 'lzop';
5218     } elsif ($archive =~ m/\.vma$/) {
5219         $format = 'vma' if !$format;
5220     } elsif ($archive =~ m/\.vma\.gz$/) {
5221         $format = 'vma' if !$format;
5222         $comp = 'gzip';
5223     } elsif ($archive =~ m/\.vma\.lzo$/) {
5224         $format = 'vma' if !$format;
5225         $comp = 'lzop';
5226     } else {
5227         $format = 'vma' if !$format; # default
5228     }
5229
5230     # try to detect archive format
5231     if ($format eq 'tar') {
5232         return restore_tar_archive($archive, $vmid, $user, $opts);
5233     } else {
5234         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5235     }
5236 }
5237
5238 sub restore_update_config_line {
5239     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5240
5241     return if $line =~ m/^\#qmdump\#/;
5242     return if $line =~ m/^\#vzdump\#/;
5243     return if $line =~ m/^lock:/;
5244     return if $line =~ m/^unused\d+:/;
5245     return if $line =~ m/^parent:/;
5246     return if $line =~ m/^template:/; # restored VM is never a template
5247
5248     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5249         # try to convert old 1.X settings
5250         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5251         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5252             my ($model, $macaddr) = split(/\=/, $devconfig);
5253             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5254             my $net = {
5255                 model => $model,
5256                 bridge => "vmbr$ind",
5257                 macaddr => $macaddr,
5258             };
5259             my $netstr = print_net($net);
5260
5261             print $outfd "net$cookie->{netcount}: $netstr\n";
5262             $cookie->{netcount}++;
5263         }
5264     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5265         my ($id, $netstr) = ($1, $2);
5266         my $net = parse_net($netstr);
5267         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5268         $netstr = print_net($net);
5269         print $outfd "$id: $netstr\n";
5270     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5271         my $virtdev = $1;
5272         my $value = $3;
5273         my $di = parse_drive($virtdev, $value);
5274         if (defined($di->{backup}) && !$di->{backup}) {
5275             print $outfd "#$line";
5276         } elsif ($map->{$virtdev}) {
5277             delete $di->{format}; # format can change on restore
5278             $di->{file} = $map->{$virtdev};
5279             $value = print_drive($vmid, $di);
5280             print $outfd "$virtdev: $value\n";
5281         } else {
5282             print $outfd $line;
5283         }
5284     } else {
5285         print $outfd $line;
5286     }
5287 }
5288
5289 sub scan_volids {
5290     my ($cfg, $vmid) = @_;
5291
5292     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5293
5294     my $volid_hash = {};
5295     foreach my $storeid (keys %$info) {
5296         foreach my $item (@{$info->{$storeid}}) {
5297             next if !($item->{volid} && $item->{size});
5298             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5299             $volid_hash->{$item->{volid}} = $item;
5300         }
5301     }
5302
5303     return $volid_hash;
5304 }
5305
5306 sub is_volume_in_use {
5307     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5308
5309     my $path = PVE::Storage::path($storecfg, $volid);
5310
5311     my $scan_config = sub {
5312         my ($cref, $snapname) = @_;
5313
5314         foreach my $key (keys %$cref) {
5315             my $value = $cref->{$key};
5316             if (is_valid_drivename($key)) {
5317                 next if $skip_drive && $key eq $skip_drive;
5318                 my $drive = parse_drive($key, $value);
5319                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5320                 return 1 if $volid eq $drive->{file};
5321                 if ($drive->{file} =~ m!^/!) {
5322                     return 1 if $drive->{file} eq $path;
5323                 } else {
5324                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5325                     next if !$storeid;
5326                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5327                     next if !$scfg;
5328                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5329                 }
5330             }
5331         }
5332
5333         return 0;
5334     };
5335
5336     return 1 if &$scan_config($conf);
5337
5338     undef $skip_drive;
5339
5340     foreach my $snapname (keys %{$conf->{snapshots}}) {
5341         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5342     }
5343
5344     return 0;
5345 }
5346
5347 sub update_disksize {
5348     my ($vmid, $conf, $volid_hash) = @_;
5349
5350     my $changes;
5351
5352     my $used = {};
5353
5354     # Note: it is allowed to define multiple storages with same path (alias), so
5355     # we need to check both 'volid' and real 'path' (two different volid can point
5356     # to the same path).
5357
5358     my $usedpath = {};
5359
5360     # update size info
5361     foreach my $opt (keys %$conf) {
5362         if (is_valid_drivename($opt)) {
5363             my $drive = parse_drive($opt, $conf->{$opt});
5364             my $volid = $drive->{file};
5365             next if !$volid;
5366
5367             $used->{$volid} = 1;
5368             if ($volid_hash->{$volid} &&
5369                 (my $path = $volid_hash->{$volid}->{path})) {
5370                 $usedpath->{$path} = 1;
5371             }
5372
5373             next if drive_is_cdrom($drive);
5374             next if !$volid_hash->{$volid};
5375
5376             $drive->{size} = $volid_hash->{$volid}->{size};
5377             my $new = print_drive($vmid, $drive);
5378             if ($new ne $conf->{$opt}) {
5379                 $changes = 1;
5380                 $conf->{$opt} = $new;
5381             }
5382         }
5383     }
5384
5385     # remove 'unusedX' entry if volume is used
5386     foreach my $opt (keys %$conf) {
5387         next if $opt !~ m/^unused\d+$/;
5388         my $volid = $conf->{$opt};
5389         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5390         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5391             $changes = 1;
5392             delete $conf->{$opt};
5393         }
5394     }
5395
5396     foreach my $volid (sort keys %$volid_hash) {
5397         next if $volid =~ m/vm-$vmid-state-/;
5398         next if $used->{$volid};
5399         my $path = $volid_hash->{$volid}->{path};
5400         next if !$path; # just to be sure
5401         next if $usedpath->{$path};
5402         $changes = 1;
5403         PVE::QemuConfig->add_unused_volume($conf, $volid);
5404         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5405     }
5406
5407     return $changes;
5408 }
5409
5410 sub rescan {
5411     my ($vmid, $nolock) = @_;
5412
5413     my $cfg = PVE::Storage::config();
5414
5415     my $volid_hash = scan_volids($cfg, $vmid);
5416
5417     my $updatefn =  sub {
5418         my ($vmid) = @_;
5419
5420         my $conf = PVE::QemuConfig->load_config($vmid);
5421
5422         PVE::QemuConfig->check_lock($conf);
5423
5424         my $vm_volids = {};
5425         foreach my $volid (keys %$volid_hash) {
5426             my $info = $volid_hash->{$volid};
5427             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5428         }
5429
5430         my $changes = update_disksize($vmid, $conf, $vm_volids);
5431
5432         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5433     };
5434
5435     if (defined($vmid)) {
5436         if ($nolock) {
5437             &$updatefn($vmid);
5438         } else {
5439             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5440         }
5441     } else {
5442         my $vmlist = config_list();
5443         foreach my $vmid (keys %$vmlist) {
5444             if ($nolock) {
5445                 &$updatefn($vmid);
5446             } else {
5447                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5448             }
5449         }
5450     }
5451 }
5452
5453 sub restore_vma_archive {
5454     my ($archive, $vmid, $user, $opts, $comp) = @_;
5455
5456     my $input = $archive eq '-' ? "<&STDIN" : undef;
5457     my $readfrom = $archive;
5458
5459     my $uncomp = '';
5460     if ($comp) {
5461         $readfrom = '-';
5462         my $qarchive = PVE::Tools::shellquote($archive);
5463         if ($comp eq 'gzip') {
5464             $uncomp = "zcat $qarchive|";
5465         } elsif ($comp eq 'lzop') {
5466             $uncomp = "lzop -d -c $qarchive|";
5467         } else {
5468             die "unknown compression method '$comp'\n";
5469         }
5470
5471     }
5472
5473     my $tmpdir = "/var/tmp/vzdumptmp$$";
5474     rmtree $tmpdir;
5475
5476     # disable interrupts (always do cleanups)
5477     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5478         warn "got interrupt - ignored\n";
5479     };
5480
5481     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5482     POSIX::mkfifo($mapfifo, 0600);
5483     my $fifofh;
5484
5485     my $openfifo = sub {
5486         open($fifofh, '>', $mapfifo) || die $!;
5487     };
5488
5489     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5490
5491     my $oldtimeout;
5492     my $timeout = 5;
5493
5494     my $devinfo = {};
5495
5496     my $rpcenv = PVE::RPCEnvironment::get();
5497
5498     my $conffile = PVE::QemuConfig->config_file($vmid);
5499     my $tmpfn = "$conffile.$$.tmp";
5500
5501     # Note: $oldconf is undef if VM does not exists
5502     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5503     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5504
5505     my $print_devmap = sub {
5506         my $virtdev_hash = {};
5507
5508         my $cfgfn = "$tmpdir/qemu-server.conf";
5509
5510         # we can read the config - that is already extracted
5511         my $fh = IO::File->new($cfgfn, "r") ||
5512             "unable to read qemu-server.conf - $!\n";
5513
5514         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5515         if (-f $fwcfgfn) {
5516             my $pve_firewall_dir = '/etc/pve/firewall';
5517             mkdir $pve_firewall_dir; # make sure the dir exists
5518             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5519         }
5520
5521         while (defined(my $line = <$fh>)) {
5522             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5523                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5524                 die "archive does not contain data for drive '$virtdev'\n"
5525                     if !$devinfo->{$devname};
5526                 if (defined($opts->{storage})) {
5527                     $storeid = $opts->{storage} || 'local';
5528                 } elsif (!$storeid) {
5529                     $storeid = 'local';
5530                 }
5531                 $format = 'raw' if !$format;
5532                 $devinfo->{$devname}->{devname} = $devname;
5533                 $devinfo->{$devname}->{virtdev} = $virtdev;
5534                 $devinfo->{$devname}->{format} = $format;
5535                 $devinfo->{$devname}->{storeid} = $storeid;
5536
5537                 # check permission on storage
5538                 my $pool = $opts->{pool}; # todo: do we need that?
5539                 if ($user ne 'root@pam') {
5540                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5541                 }
5542
5543                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5544             }
5545         }
5546
5547         foreach my $devname (keys %$devinfo) {
5548             die "found no device mapping information for device '$devname'\n"
5549                 if !$devinfo->{$devname}->{virtdev};
5550         }
5551
5552         my $cfg = PVE::Storage::config();
5553
5554         # create empty/temp config
5555         if ($oldconf) {
5556             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5557             foreach_drive($oldconf, sub {
5558                 my ($ds, $drive) = @_;
5559
5560                 return if drive_is_cdrom($drive);
5561
5562                 my $volid = $drive->{file};
5563
5564                 return if !$volid || $volid =~ m|^/|;
5565
5566                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5567                 return if !$path || !$owner || ($owner != $vmid);
5568
5569                 # Note: only delete disk we want to restore
5570                 # other volumes will become unused
5571                 if ($virtdev_hash->{$ds}) {
5572                     PVE::Storage::vdisk_free($cfg, $volid);
5573                 }
5574             });
5575
5576             # delete vmstate files
5577             # since after the restore we have no snapshots anymore
5578             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5579                 my $snap = $oldconf->{snapshots}->{$snapname};
5580                 if ($snap->{vmstate}) {
5581                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5582                     if (my $err = $@) {
5583                         warn $err;
5584                     }
5585                 }
5586             }
5587         }
5588
5589         my $map = {};
5590         foreach my $virtdev (sort keys %$virtdev_hash) {
5591             my $d = $virtdev_hash->{$virtdev};
5592             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5593             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5594
5595             # test if requested format is supported
5596             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5597             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5598             $d->{format} = $defFormat if !$supported;
5599
5600             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5601                                                   $d->{format}, undef, $alloc_size);
5602             print STDERR "new volume ID is '$volid'\n";
5603             $d->{volid} = $volid;
5604             my $path = PVE::Storage::path($cfg, $volid);
5605
5606             PVE::Storage::activate_volumes($cfg,[$volid]);
5607
5608             my $write_zeros = 1;
5609             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5610                 $write_zeros = 0;
5611             }
5612
5613             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5614
5615             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5616             $map->{$virtdev} = $volid;
5617         }
5618
5619         $fh->seek(0, 0) || die "seek failed - $!\n";
5620
5621         my $outfd = new IO::File ($tmpfn, "w") ||
5622             die "unable to write config for VM $vmid\n";
5623
5624         my $cookie = { netcount => 0 };
5625         while (defined(my $line = <$fh>)) {
5626             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5627         }
5628
5629         $fh->close();
5630         $outfd->close();
5631     };
5632
5633     eval {
5634         # enable interrupts
5635         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5636             die "interrupted by signal\n";
5637         };
5638         local $SIG{ALRM} = sub { die "got timeout\n"; };
5639
5640         $oldtimeout = alarm($timeout);
5641
5642         my $parser = sub {
5643             my $line = shift;
5644
5645             print "$line\n";
5646
5647             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5648                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5649                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5650             } elsif ($line =~ m/^CTIME: /) {
5651                 # we correctly received the vma config, so we can disable
5652                 # the timeout now for disk allocation (set to 10 minutes, so
5653                 # that we always timeout if something goes wrong)
5654                 alarm(600);
5655                 &$print_devmap();
5656                 print $fifofh "done\n";
5657                 my $tmp = $oldtimeout || 0;
5658                 $oldtimeout = undef;
5659                 alarm($tmp);
5660                 close($fifofh);
5661             }
5662         };
5663
5664         print "restore vma archive: $cmd\n";
5665         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5666     };
5667     my $err = $@;
5668
5669     alarm($oldtimeout) if $oldtimeout;
5670
5671     my $vollist = [];
5672     foreach my $devname (keys %$devinfo) {
5673         my $volid = $devinfo->{$devname}->{volid};
5674         push @$vollist, $volid if $volid;
5675     }
5676
5677     my $cfg = PVE::Storage::config();
5678     PVE::Storage::deactivate_volumes($cfg, $vollist);
5679
5680     unlink $mapfifo;
5681
5682     if ($err) {
5683         rmtree $tmpdir;
5684         unlink $tmpfn;
5685
5686         foreach my $devname (keys %$devinfo) {
5687             my $volid = $devinfo->{$devname}->{volid};
5688             next if !$volid;
5689             eval {
5690                 if ($volid =~ m|^/|) {
5691                     unlink $volid || die 'unlink failed\n';
5692                 } else {
5693                     PVE::Storage::vdisk_free($cfg, $volid);
5694                 }
5695                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5696             };
5697             print STDERR "unable to cleanup '$volid' - $@" if $@;
5698         }
5699         die $err;
5700     }
5701
5702     rmtree $tmpdir;
5703
5704     rename($tmpfn, $conffile) ||
5705         die "unable to commit configuration file '$conffile'\n";
5706
5707     PVE::Cluster::cfs_update(); # make sure we read new file
5708
5709     eval { rescan($vmid, 1); };
5710     warn $@ if $@;
5711 }
5712
5713 sub restore_tar_archive {
5714     my ($archive, $vmid, $user, $opts) = @_;
5715
5716     if ($archive ne '-') {
5717         my $firstfile = tar_archive_read_firstfile($archive);
5718         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5719             if $firstfile ne 'qemu-server.conf';
5720     }
5721
5722     my $storecfg = PVE::Storage::config();
5723
5724     # destroy existing data - keep empty config
5725     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5726     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5727
5728     my $tocmd = "/usr/lib/qemu-server/qmextract";
5729
5730     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5731     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5732     $tocmd .= ' --prealloc' if $opts->{prealloc};
5733     $tocmd .= ' --info' if $opts->{info};
5734
5735     # tar option "xf" does not autodetect compression when read from STDIN,
5736     # so we pipe to zcat
5737     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5738         PVE::Tools::shellquote("--to-command=$tocmd");
5739
5740     my $tmpdir = "/var/tmp/vzdumptmp$$";
5741     mkpath $tmpdir;
5742
5743     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5744     local $ENV{VZDUMP_VMID} = $vmid;
5745     local $ENV{VZDUMP_USER} = $user;
5746
5747     my $conffile = PVE::QemuConfig->config_file($vmid);
5748     my $tmpfn = "$conffile.$$.tmp";
5749
5750     # disable interrupts (always do cleanups)
5751     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5752         print STDERR "got interrupt - ignored\n";
5753     };
5754
5755     eval {
5756         # enable interrupts
5757         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5758             die "interrupted by signal\n";
5759         };
5760
5761         if ($archive eq '-') {
5762             print "extracting archive from STDIN\n";
5763             run_command($cmd, input => "<&STDIN");
5764         } else {
5765             print "extracting archive '$archive'\n";
5766             run_command($cmd);
5767         }
5768
5769         return if $opts->{info};
5770
5771         # read new mapping
5772         my $map = {};
5773         my $statfile = "$tmpdir/qmrestore.stat";
5774         if (my $fd = IO::File->new($statfile, "r")) {
5775             while (defined (my $line = <$fd>)) {
5776                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5777                     $map->{$1} = $2 if $1;
5778                 } else {
5779                     print STDERR "unable to parse line in statfile - $line\n";
5780                 }
5781             }
5782             $fd->close();
5783         }
5784
5785         my $confsrc = "$tmpdir/qemu-server.conf";
5786
5787         my $srcfd = new IO::File($confsrc, "r") ||
5788             die "unable to open file '$confsrc'\n";
5789
5790         my $outfd = new IO::File ($tmpfn, "w") ||
5791             die "unable to write config for VM $vmid\n";
5792
5793         my $cookie = { netcount => 0 };
5794         while (defined (my $line = <$srcfd>)) {
5795             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5796         }
5797
5798         $srcfd->close();
5799         $outfd->close();
5800     };
5801     my $err = $@;
5802
5803     if ($err) {
5804
5805         unlink $tmpfn;
5806
5807         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5808
5809         die $err;
5810     }
5811
5812     rmtree $tmpdir;
5813
5814     rename $tmpfn, $conffile ||
5815         die "unable to commit configuration file '$conffile'\n";
5816
5817     PVE::Cluster::cfs_update(); # make sure we read new file
5818
5819     eval { rescan($vmid, 1); };
5820     warn $@ if $@;
5821 };
5822
5823 sub foreach_writable_storage {
5824     my ($conf, $func) = @_;
5825
5826     my $sidhash = {};
5827
5828     foreach my $ds (keys %$conf) {
5829         next if !is_valid_drivename($ds);
5830
5831         my $drive = parse_drive($ds, $conf->{$ds});
5832         next if !$drive;
5833         next if drive_is_cdrom($drive);
5834
5835         my $volid = $drive->{file};
5836
5837         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5838         $sidhash->{$sid} = $sid if $sid;
5839     }
5840
5841     foreach my $sid (sort keys %$sidhash) {
5842         &$func($sid);
5843     }
5844 }
5845
5846 sub do_snapshots_with_qemu {
5847     my ($storecfg, $volid) = @_;
5848
5849     my $storage_name = PVE::Storage::parse_volume_id($volid);
5850
5851     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5852         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5853         return 1;
5854     }
5855
5856     if ($volid =~ m/\.(qcow2|qed)$/){
5857         return 1;
5858     }
5859
5860     return undef;
5861 }
5862
5863 sub qga_check_running {
5864     my ($vmid) = @_;
5865
5866     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5867     if ($@) {
5868         warn "Qemu Guest Agent are not running - $@";
5869         return 0;
5870     }
5871     return 1;
5872 }
5873
5874 sub template_create {
5875     my ($vmid, $conf, $disk) = @_;
5876
5877     my $storecfg = PVE::Storage::config();
5878
5879     foreach_drive($conf, sub {
5880         my ($ds, $drive) = @_;
5881
5882         return if drive_is_cdrom($drive);
5883         return if $disk && $ds ne $disk;
5884
5885         my $volid = $drive->{file};
5886         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5887
5888         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5889         $drive->{file} = $voliddst;
5890         $conf->{$ds} = print_drive($vmid, $drive);
5891         PVE::QemuConfig->write_config($vmid, $conf);
5892     });
5893 }
5894
5895 sub qemu_img_convert {
5896     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5897
5898     my $storecfg = PVE::Storage::config();
5899     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5900     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5901
5902     if ($src_storeid && $dst_storeid) {
5903
5904         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5905
5906         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5907         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5908
5909         my $src_format = qemu_img_format($src_scfg, $src_volname);
5910         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5911
5912         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5913         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5914
5915         my $cmd = [];
5916         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5917         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5918         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5919         if ($is_zero_initialized) {
5920             push @$cmd, "zeroinit:$dst_path";
5921         } else {
5922             push @$cmd, $dst_path;
5923         }
5924
5925         my $parser = sub {
5926             my $line = shift;
5927             if($line =~ m/\((\S+)\/100\%\)/){
5928                 my $percent = $1;
5929                 my $transferred = int($size * $percent / 100);
5930                 my $remaining = $size - $transferred;
5931
5932                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5933             }
5934
5935         };
5936
5937         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5938         my $err = $@;
5939         die "copy failed: $err" if $err;
5940     }
5941 }
5942
5943 sub qemu_img_format {
5944     my ($scfg, $volname) = @_;
5945
5946     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5947         return $1;
5948     } else {
5949         return "raw";
5950     }
5951 }
5952
5953 sub qemu_drive_mirror {
5954     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5955
5956     my $storecfg = PVE::Storage::config();
5957     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5958
5959     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5960
5961     my $format = qemu_img_format($dst_scfg, $dst_volname);
5962
5963     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5964
5965     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5966
5967     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5968     $opts->{format} = $format if $format;
5969
5970     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5971
5972     my $finish_job = sub {
5973         while (1) {
5974             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5975             my $stat = @$stats[0];
5976             last if !$stat;
5977             sleep 1;
5978         }
5979     };
5980
5981     eval {
5982     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5983         while (1) {
5984             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5985             my $stat = @$stats[0];
5986             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5987             die "error job is not mirroring" if $stat->{type} ne "mirror";
5988
5989             my $busy = $stat->{busy};
5990             my $ready = $stat->{ready};
5991
5992             if (my $total = $stat->{len}) {
5993                 my $transferred = $stat->{offset} || 0;
5994                 my $remaining = $total - $transferred;
5995                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5996
5997                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5998             }
5999
6000
6001             if ($stat->{ready} eq 'true') {
6002
6003                 last if $vmiddst != $vmid;
6004
6005                 # try to switch the disk if source and destination are on the same guest
6006                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6007                 if (!$@) {
6008                     &$finish_job();
6009                     last;
6010                 }
6011                 die $@ if $@ !~ m/cannot be completed/;
6012             }
6013             sleep 1;
6014         }
6015
6016
6017     };
6018     my $err = $@;
6019
6020     my $cancel_job = sub {
6021         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6022         &$finish_job();
6023     };
6024
6025     if ($err) {
6026         eval { &$cancel_job(); };
6027         die "mirroring error: $err";
6028     }
6029
6030     if ($vmiddst != $vmid) {
6031         # if we clone a disk for a new target vm, we don't switch the disk
6032         &$cancel_job(); # so we call block-job-cancel
6033     }
6034 }
6035
6036 sub clone_disk {
6037     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6038         $newvmid, $storage, $format, $full, $newvollist) = @_;
6039
6040     my $newvolid;
6041
6042     if (!$full) {
6043         print "create linked clone of drive $drivename ($drive->{file})\n";
6044         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6045         push @$newvollist, $newvolid;
6046     } else {
6047         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6048         $storeid = $storage if $storage;
6049
6050         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6051         if (!$format) {
6052             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6053             $format = qemu_img_format($scfg, $volname);
6054         }
6055
6056         # test if requested format is supported - else use default
6057         my $supported = grep { $_ eq $format } @$validFormats;
6058         $format = $defFormat if !$supported;
6059
6060         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6061
6062         print "create full clone of drive $drivename ($drive->{file})\n";
6063         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6064         push @$newvollist, $newvolid;
6065
6066         PVE::Storage::activate_volumes($storecfg, $newvollist);
6067
6068         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6069         if (!$running || $snapname) {
6070             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6071         } else {
6072             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
6073         }
6074     }
6075
6076     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6077
6078     my $disk = $drive;
6079     $disk->{format} = undef;
6080     $disk->{file} = $newvolid;
6081     $disk->{size} = $size;
6082
6083     return $disk;
6084 }
6085
6086 # this only works if VM is running
6087 sub get_current_qemu_machine {
6088     my ($vmid) = @_;
6089
6090     my $cmd = { execute => 'query-machines', arguments => {} };
6091     my $res = vm_qmp_command($vmid, $cmd);
6092
6093     my ($current, $default);
6094     foreach my $e (@$res) {
6095         $default = $e->{name} if $e->{'is-default'};
6096         $current = $e->{name} if $e->{'is-current'};
6097     }
6098
6099     # fallback to the default machine if current is not supported by qemu
6100     return $current || $default || 'pc';
6101 }
6102
6103 sub qemu_machine_feature_enabled {
6104     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6105
6106     my $current_major;
6107     my $current_minor;
6108
6109     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6110
6111         $current_major = $3;
6112         $current_minor = $4;
6113
6114     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6115
6116         $current_major = $1;
6117         $current_minor = $2;
6118     }
6119
6120     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6121
6122
6123 }
6124
6125 sub qemu_machine_pxe {
6126     my ($vmid, $conf, $machine) = @_;
6127
6128     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6129
6130     foreach my $opt (keys %$conf) {
6131         next if $opt !~ m/^net(\d+)$/;
6132         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6133         next if !$net;
6134         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6135         return $machine.".pxe" if $romfile =~ m/pxe/;
6136         last;
6137     }
6138
6139     return $machine;
6140 }
6141
6142 sub qemu_use_old_bios_files {
6143     my ($machine_type) = @_;
6144
6145     return if !$machine_type;
6146
6147     my $use_old_bios_files = undef;
6148
6149     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6150         $machine_type = $1;
6151         $use_old_bios_files = 1;
6152     } else {
6153         my $kvmver = kvm_user_version();
6154         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6155         # load new efi bios files on migration. So this hack is required to allow
6156         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6157         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6158         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6159     }
6160
6161     return ($use_old_bios_files, $machine_type);
6162 }
6163
6164 sub lspci {
6165
6166     my $devices = {};
6167
6168     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6169             my (undef, $id, $function) = @_;
6170             my $res = { id => $id, function => $function};
6171             push @{$devices->{$id}}, $res;
6172     });
6173
6174     return $devices;
6175 }
6176
6177 sub vm_iothreads_list {
6178     my ($vmid) = @_;
6179
6180     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6181
6182     my $iothreads = {};
6183     foreach my $iothread (@$res) {
6184         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6185     }
6186
6187     return $iothreads;
6188 }
6189
6190 sub scsihw_infos {
6191     my ($conf, $drive) = @_;
6192
6193     my $maxdev = 0;
6194
6195     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6196         $maxdev = 7;
6197     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6198         $maxdev = 1;
6199     } else {
6200         $maxdev = 256;
6201     }
6202
6203     my $controller = int($drive->{index} / $maxdev);
6204     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6205
6206     return ($maxdev, $controller, $controller_prefix);
6207 }
6208
6209 # bash completion helper
6210
6211 sub complete_backup_archives {
6212     my ($cmdname, $pname, $cvalue) = @_;
6213
6214     my $cfg = PVE::Storage::config();
6215
6216     my $storeid;
6217
6218     if ($cvalue =~ m/^([^:]+):/) {
6219         $storeid = $1;
6220     }
6221
6222     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6223
6224     my $res = [];
6225     foreach my $id (keys %$data) {
6226         foreach my $item (@{$data->{$id}}) {
6227             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6228             push @$res, $item->{volid} if defined($item->{volid});
6229         }
6230     }
6231
6232     return $res;
6233 }
6234
6235 my $complete_vmid_full = sub {
6236     my ($running) = @_;
6237
6238     my $idlist = vmstatus();
6239
6240     my $res = [];
6241
6242     foreach my $id (keys %$idlist) {
6243         my $d = $idlist->{$id};
6244         if (defined($running)) {
6245             next if $d->{template};
6246             next if $running && $d->{status} ne 'running';
6247             next if !$running && $d->{status} eq 'running';
6248         }
6249         push @$res, $id;
6250
6251     }
6252     return $res;
6253 };
6254
6255 sub complete_vmid {
6256     return &$complete_vmid_full();
6257 }
6258
6259 sub complete_vmid_stopped {
6260     return &$complete_vmid_full(0);
6261 }
6262
6263 sub complete_vmid_running {
6264     return &$complete_vmid_full(1);
6265 }
6266
6267 sub complete_storage {
6268
6269     my $cfg = PVE::Storage::config();
6270     my $ids = $cfg->{ids};
6271
6272     my $res = [];
6273     foreach my $sid (keys %$ids) {
6274         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6275         next if !$ids->{$sid}->{content}->{images};
6276         push @$res, $sid;
6277     }
6278
6279     return $res;
6280 }
6281
6282 1;