PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35 use URI::Escape;
  36
  37 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  38
  39 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  40
  41 # Note about locking: we use flock on the config file protect
  42 # against concurent actions.
  43 # Aditionaly, we have a 'lock' setting in the config file. This
  44 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  45 # allowed when such lock is set. But you can ignore this kind of
  46 # lock with the --skiplock flag.
  47
  48 cfs_register_file('/qemu-server/',
  49                   \&parse_vm_config,
  50                   \&write_vm_config);
  51
  52 PVE::JSONSchema::register_standard_option('skiplock', {
  53     description => "Ignore locks - only root is allowed to use this option.",
  54     type => 'boolean',
  55     optional => 1,
  56 });
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 #no warnings 'redefine';
  72
  73 sub cgroups_write {
  74    my ($controller, $vmid, $option, $value) = @_;
  75
  76    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  77    PVE::ProcFSTools::write_proc_entry($path, $value);
  78
  79 }
  80
  81 my $nodename = PVE::INotify::nodename();
  82
  83 mkdir "/etc/pve/nodes/$nodename";
  84 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  85 mkdir $confdir;
  86
  87 my $var_run_tmpdir = "/var/run/qemu-server";
  88 mkdir $var_run_tmpdir;
  89
  90 my $lock_dir = "/var/lock/qemu-server";
  91 mkdir $lock_dir;
  92
  93 my $pcisysfs = "/sys/bus/pci";
  94
  95 my $cpu_fmt = {
  96     cputype => {
  97         description => "Emulated CPU type.",
  98         type => 'string',
  99         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 100         format_description => 'cputype',
 101         default => 'kvm64',
 102         default_key => 1,
 103     },
 104     hidden => {
 105         description => "Do not identify as a KVM virtual machine.",
 106         type => 'boolean',
 107         optional => 1,
 108         default => 0
 109     },
 110 };
 111
 112 my $watchdog_fmt = {
 113     model => {
 114         default_key => 1,
 115         type => 'string',
 116         enum => [qw(i6300esb ib700)],
 117         description => "Watchdog type to emulate.",
 118         default => 'i6300esb',
 119         optional => 1,
 120     },
 121     action => {
 122         type => 'string',
 123         enum => [qw(reset shutdown poweroff pause debug none)],
 124         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 125         optional => 1,
 126     },
 127 };
 128 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 129
 130 my $confdesc = {
 131     onboot => {
 132         optional => 1,
 133         type => 'boolean',
 134         description => "Specifies whether a VM will be started during system bootup.",
 135         default => 0,
 136     },
 137     autostart => {
 138         optional => 1,
 139         type => 'boolean',
 140         description => "Automatic restart after crash (currently ignored).",
 141         default => 0,
 142     },
 143     hotplug => {
 144         optional => 1,
 145         type => 'string', format => 'pve-hotplug-features',
 146         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 147         default => 'network,disk,usb',
 148     },
 149     reboot => {
 150         optional => 1,
 151         type => 'boolean',
 152         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 153         default => 1,
 154     },
 155     lock => {
 156         optional => 1,
 157         type => 'string',
 158         description => "Lock/unlock the VM.",
 159         enum => [qw(migrate backup snapshot rollback)],
 160     },
 161     cpulimit => {
 162         optional => 1,
 163         type => 'number',
 164         description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 165         minimum => 0,
 166         maximum => 128,
 167         default => 0,
 168     },
 169     cpuunits => {
 170         optional => 1,
 171         type => 'integer',
 172         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 173         minimum => 0,
 174         maximum => 500000,
 175         default => 1000,
 176     },
 177     memory => {
 178         optional => 1,
 179         type => 'integer',
 180         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 181         minimum => 16,
 182         default => 512,
 183     },
 184     balloon => {
 185         optional => 1,
 186         type => 'integer',
 187         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 188         minimum => 0,
 189     },
 190     shares => {
 191         optional => 1,
 192         type => 'integer',
 193         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 194         minimum => 0,
 195         maximum => 50000,
 196         default => 1000,
 197     },
 198     keyboard => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 202         enum => PVE::Tools::kvmkeymaplist(),
 203         default => 'en-us',
 204     },
 205     name => {
 206         optional => 1,
 207         type => 'string', format => 'dns-name',
 208         description => "Set a name for the VM. Only used on the configuration web interface.",
 209     },
 210     scsihw => {
 211         optional => 1,
 212         type => 'string',
 213         description => "scsi controller model",
 214         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 215         default => 'lsi',
 216     },
 217     description => {
 218         optional => 1,
 219         type => 'string',
 220         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 221     },
 222     ostype => {
 223         optional => 1,
 224         type => 'string',
 225         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 226         description => <<EODESC,
 227 Used to enable special optimization/features for specific
 228 operating systems:
 229
 230 other  => unspecified OS
 231 wxp    => Microsoft Windows XP
 232 w2k    => Microsoft Windows 2000
 233 w2k3   => Microsoft Windows 2003
 234 w2k8   => Microsoft Windows 2008
 235 wvista => Microsoft Windows Vista
 236 win7   => Microsoft Windows 7
 237 win8   => Microsoft Windows 8/2012
 238 l24    => Linux 2.4 Kernel
 239 l26    => Linux 2.6/3.X Kernel
 240 solaris => solaris/opensolaris/openindiania kernel
 241
 242 other|l24|l26|solaris                       ... no special behaviour
 243 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 244 EODESC
 245     },
 246     boot => {
 247         optional => 1,
 248         type => 'string',
 249         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 250         pattern => '[acdn]{1,4}',
 251         default => 'cdn',
 252     },
 253     bootdisk => {
 254         optional => 1,
 255         type => 'string', format => 'pve-qm-bootdisk',
 256         description => "Enable booting from specified disk.",
 257         pattern => '(ide|sata|scsi|virtio)\d+',
 258     },
 259     smp => {
 260         optional => 1,
 261         type => 'integer',
 262         description => "The number of CPUs. Please use option -sockets instead.",
 263         minimum => 1,
 264         default => 1,
 265     },
 266     sockets => {
 267         optional => 1,
 268         type => 'integer',
 269         description => "The number of CPU sockets.",
 270         minimum => 1,
 271         default => 1,
 272     },
 273     cores => {
 274         optional => 1,
 275         type => 'integer',
 276         description => "The number of cores per socket.",
 277         minimum => 1,
 278         default => 1,
 279     },
 280     numa => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Enable/disable NUMA.",
 284         default => 0,
 285     },
 286     vcpus => {
 287         optional => 1,
 288         type => 'integer',
 289         description => "Number of hotplugged vcpus.",
 290         minimum => 1,
 291         default => 0,
 292     },
 293     acpi => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Enable/disable ACPI.",
 297         default => 1,
 298     },
 299     agent => {
 300         optional => 1,
 301         type => 'boolean',
 302         description => "Enable/disable Qemu GuestAgent.",
 303         default => 0,
 304     },
 305     kvm => {
 306         optional => 1,
 307         type => 'boolean',
 308         description => "Enable/disable KVM hardware virtualization.",
 309         default => 1,
 310     },
 311     tdf => {
 312         optional => 1,
 313         type => 'boolean',
 314         description => "Enable/disable time drift fix.",
 315         default => 0,
 316     },
 317     localtime => {
 318         optional => 1,
 319         type => 'boolean',
 320         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 321     },
 322     freeze => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 326     },
 327     vga => {
 328         optional => 1,
 329         type => 'string',
 330         description => "Select the VGA type. If you want to use high resolution" .
 331             " modes (>= 1280x1024x16) then you should use the options " .
 332             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 333             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 334             "display sever. For win* OS you can select how many independent " .
 335             "displays you want, Linux guests can add displays them self. " .
 336             "You can also run without any graphic card, using a serial device" .
 337             " as terminal.",
 338         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 339     },
 340     watchdog => {
 341         optional => 1,
 342         type => 'string', format => 'pve-qm-watchdog',
 343         description => "Create a virtual hardware watchdog device. Once enabled" .
 344             " (by a guest action), the watchdog must be periodically polled " .
 345             "by an agent inside the guest or else the watchdog will reset " .
 346             "the guest (or execute the respective action specified)",
 347     },
 348     startdate => {
 349         optional => 1,
 350         type => 'string',
 351         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 352         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 353         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 354         default => 'now',
 355     },
 356     startup =>  get_standard_option('pve-startup-order'),
 357     template => {
 358         optional => 1,
 359         type => 'boolean',
 360         description => "Enable/disable Template.",
 361         default => 0,
 362     },
 363     args => {
 364         optional => 1,
 365         type => 'string',
 366         description => <<EODESCR,
 367 NOTE: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 368
 369 args: -no-reboot -no-hpet
 370 EODESCR
 371     },
 372     tablet => {
 373         optional => 1,
 374         type => 'boolean',
 375         default => 1,
 376         description => "Enable/disable the USB tablet device. This device is " .
 377             "usually needed to allow absolute mouse positioning with VNC. " .
 378             "Else the mouse runs out of sync with normal VNC clients. " .
 379             "If you're running lots of console-only guests on one host, " .
 380             "you may consider disabling this to save some context switches. " .
 381             "This is turned off by default if you use spice (-vga=qxl).",
 382     },
 383     migrate_speed => {
 384         optional => 1,
 385         type => 'integer',
 386         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 387         minimum => 0,
 388         default => 0,
 389     },
 390     migrate_downtime => {
 391         optional => 1,
 392         type => 'number',
 393         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 394         minimum => 0,
 395         default => 0.1,
 396     },
 397     cdrom => {
 398         optional => 1,
 399         type => 'string', format => 'pve-qm-drive',
 400         typetext => 'volume',
 401         description => "This is an alias for option -ide2",
 402     },
 403     cpu => {
 404         optional => 1,
 405         description => "Emulated CPU type.",
 406         type => 'string',
 407         format => $cpu_fmt,
 408     },
 409     parent => get_standard_option('pve-snapshot-name', {
 410         optional => 1,
 411         description => "Parent snapshot name. This is used internally, and should not be modified.",
 412     }),
 413     snaptime => {
 414         optional => 1,
 415         description => "Timestamp for snapshots.",
 416         type => 'integer',
 417         minimum => 0,
 418     },
 419     vmstate => {
 420         optional => 1,
 421         type => 'string', format => 'pve-volume-id',
 422         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 423     },
 424     machine => {
 425         description => "Specific the Qemu machine type.",
 426         type => 'string',
 427         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 428         maxLength => 40,
 429         optional => 1,
 430     },
 431     smbios1 => {
 432         description => "Specify SMBIOS type 1 fields.",
 433         type => 'string', format => 'pve-qm-smbios1',
 434         maxLength => 256,
 435         optional => 1,
 436     },
 437     protection => {
 438         optional => 1,
 439         type => 'boolean',
 440         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 441         default => 0,
 442     },
 443     bios => {
 444         optional => 1,
 445         type => 'string',
 446         enum => [ qw(seabios ovmf) ],
 447         description => "Select BIOS implementation.",
 448         default => 'seabios',
 449     },
 450 };
 451
 452 # what about other qemu settings ?
 453 #cpu => 'string',
 454 #machine => 'string',
 455 #fda => 'file',
 456 #fdb => 'file',
 457 #mtdblock => 'file',
 458 #sd => 'file',
 459 #pflash => 'file',
 460 #snapshot => 'bool',
 461 #bootp => 'file',
 462 ##tftp => 'dir',
 463 ##smb => 'dir',
 464 #kernel => 'file',
 465 #append => 'string',
 466 #initrd => 'file',
 467 ##soundhw => 'string',
 468
 469 while (my ($k, $v) = each %$confdesc) {
 470     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 471 }
 472
 473 my $MAX_IDE_DISKS = 4;
 474 my $MAX_SCSI_DISKS = 14;
 475 my $MAX_VIRTIO_DISKS = 16;
 476 my $MAX_SATA_DISKS = 6;
 477 my $MAX_USB_DEVICES = 5;
 478 my $MAX_NETS = 32;
 479 my $MAX_UNUSED_DISKS = 8;
 480 my $MAX_HOSTPCI_DEVICES = 4;
 481 my $MAX_SERIAL_PORTS = 4;
 482 my $MAX_PARALLEL_PORTS = 3;
 483 my $MAX_NUMA = 8;
 484 my $MAX_MEM = 4194304;
 485 my $STATICMEM = 1024;
 486
 487 my $numa_fmt = {
 488     cpus => {
 489         type => "string",
 490         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 491         description => "CPUs accessing this numa node.",
 492         format_description => "id[-id];...",
 493     },
 494     memory => {
 495         type => "number",
 496         description => "Amount of memory this numa node provides.",
 497         format_description => "mb",
 498         optional => 1,
 499     },
 500     hostnodes => {
 501         type => "string",
 502         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 503         description => "host numa nodes to use",
 504         format_description => "id[-id];...",
 505         optional => 1,
 506     },
 507     policy => {
 508         type => 'string',
 509         enum => [qw(preferred bind interleave)],
 510         format_description => 'preferred|bind|interleave',
 511         description => "numa allocation policy.",
 512         optional => 1,
 513     },
 514 };
 515 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 516 my $numadesc = {
 517     optional => 1,
 518     type => 'string', format => $numa_fmt,
 519     description => "numa topology",
 520 };
 521 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 522
 523 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 524     $confdesc->{"numa$i"} = $numadesc;
 525 }
 526
 527 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 528                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 529                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 530 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 531
 532 my $net_fmt = {
 533     macaddr => {
 534         type => 'string',
 535         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 536         description => "MAC address",
 537         format_description => "XX:XX:XX:XX:XX:XX",
 538         optional => 1,
 539     },
 540     model => { alias => 'macaddr', default_key => 1 },
 541     (map { $_ => { group => 'model' } } @$nic_model_list),
 542     bridge => {
 543         type => 'string',
 544         description => 'Bridge to attach the network device to.',
 545         format_description => 'bridge',
 546         optional => 1,
 547     },
 548     queues => {
 549         type => 'integer',
 550         minimum => 0, maximum => 16,
 551         description => 'Number of packet queues to be used on the device.',
 552         format_description => 'number',
 553         optional => 1,
 554     },
 555     rate => {
 556         type => 'number',
 557         minimum => 0,
 558         description => 'Rate limit in mbps as floating point number.',
 559         format_description => 'mbps',
 560         optional => 1,
 561     },
 562     tag => {
 563         type => 'integer',
 564         minimum => 2, maximum => 4094,
 565         description => 'VLAN tag to apply to packets on this interface.',
 566         format_description => 'vlanid',
 567         optional => 1,
 568     },
 569     trunks => {
 570         type => 'string',
 571         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 572         description => 'VLAN trunks to pass through this interface.',
 573         format_description => 'id;id...',
 574         optional => 1,
 575     },
 576     firewall => {
 577         type => 'boolean',
 578         description => 'Whether this interface should be protected by the firewall.',
 579         format_description => '0|1',
 580         optional => 1,
 581     },
 582     link_down => {
 583         type => 'boolean',
 584         description => 'Whether this interface should be DISconnected (like pulling the plug).',
 585         format_description => '0|1',
 586         optional => 1,
 587     },
 588 };
 589 my $netdesc = {
 590     optional => 1,
 591     type => 'string', format => 'pve-qm-net',
 592     description => <<EODESCR,
 593 Specify network devices.
 594
 595 MODEL is one of: $nic_model_list_txt
 596
 597 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 598 automatically generated if not specified.
 599
 600 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 601
 602 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 603
 604 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 605
 606 10.0.2.2   Gateway
 607 10.0.2.3   DNS Server
 608 10.0.2.4   SMB Server
 609
 610 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 611
 612 EODESCR
 613 };
 614 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 615
 616 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 617     $confdesc->{"net$i"} = $netdesc;
 618 }
 619
 620 PVE::JSONSchema::register_format('pve-volume-id-or-none', \&verify_volume_id_or_none);
 621 sub verify_volume_id_or_none {
 622     my ($volid, $noerr) = @_;
 623
 624     return $volid if $volid eq 'none';
 625     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 626     if ($@) {
 627         return undef if $noerr;
 628         die $@;
 629     }
 630     return $volid;
 631 }
 632
 633 my $drivename_hash;
 634
 635 my %drivedesc_base = (
 636     volume => { alias => 'file' },
 637     file => {
 638         type => 'string',
 639         format => 'pve-volume-id-or-none',
 640         default_key => 1,
 641         format_description => 'volume',
 642         description => "The drive's backing volume.",
 643     },
 644     media => {
 645         type => 'string',
 646         format_description => 'cdrom|disk',
 647         enum => [qw(cdrom disk)],
 648         description => "The drive's media type.",
 649         default => 'disk',
 650         optional => 1
 651     },
 652     cyls => {
 653         type => 'integer',
 654         format_description => 'count',
 655         description => "Force the drive's physical geometry to have a specific cylinder count.",
 656         optional => 1
 657     },
 658     heads => {
 659         type => 'integer',
 660         format_description => 'count',
 661         description => "Force the drive's physical geometry to have a specific head count.",
 662         optional => 1
 663     },
 664     secs => {
 665         type => 'integer',
 666         format_description => 'count',
 667         description => "Force the drive's physical geometry to have a specific sector count.",
 668         optional => 1
 669     },
 670     trans => {
 671         type => 'string',
 672         format_description => 'none|lba|auto',
 673         enum => [qw(none lba auto)],
 674         description => "Force disk geometry bios translation mode.",
 675         optional => 1,
 676     },
 677     snapshot => {
 678         type => 'boolean',
 679         format_description => 'on|off',
 680         description => "Whether the drive should be included when making snapshots.",
 681         optional => 1,
 682     },
 683     cache => {
 684         type => 'string',
 685         format_description => 'none|writethrough|writeback|unsafe|directsync',
 686         enum => [qw(none writethrough writeback unsafe directsync)],
 687         description => "The drive's cache mode",
 688         optional => 1,
 689     },
 690     format => {
 691         type => 'string',
 692         format_description => 'drive format',
 693         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 694         description => "The drive's backing file's data format.",
 695         optional => 1,
 696     },
 697     size => {
 698         type => 'string',
 699         format => 'disk-size',
 700         description => "Disk size. This is purely informational and has no effect.",
 701         optional => 1,
 702     },
 703     backup => {
 704         type => 'boolean',
 705         format_description => 'on|off',
 706         description => "Whether the drive should be included when making backups.",
 707         optional => 1,
 708     },
 709     werror => {
 710         type => 'string',
 711         format_description => 'enospc|ignore|report|stop',
 712         enum => [qw(enospc ignore report stop)],
 713         description => 'Write error action.',
 714         optional => 1,
 715     },
 716     aio => {
 717         type => 'string',
 718         format_description => 'native|threads',
 719         enum => [qw(native threads)],
 720         description => 'AIO type to use.',
 721         optional => 1,
 722     },
 723     discard => {
 724         type => 'string',
 725         format_description => 'ignore|on',
 726         enum => [qw(ignore on)],
 727         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 728         optional => 1,
 729     },
 730     detect_zeroes => {
 731         type => 'boolean',
 732         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 733         optional => 1,
 734     },
 735     serial => {
 736         type => 'string',
 737         format => 'urlencoded',
 738         format_description => 'serial',
 739         maxLength => 20*3, # *3 since it's %xx url enoded
 740         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 741         optional => 1,
 742     }
 743 );
 744
 745 my %rerror_fmt = (
 746     rerror => {
 747         type => 'string',
 748         format_description => 'ignore|report|stop',
 749         enum => [qw(ignore report stop)],
 750         description => 'Read error action.',
 751         optional => 1,
 752     },
 753 );
 754
 755 my %iothread_fmt = ( iothread => {
 756         type => 'boolean',
 757         format_description => 'off|on',
 758         description => "Whether to use iothreads for this drive",
 759         optional => 1,
 760 });
 761
 762 my %model_fmt = (
 763     model => {
 764         type => 'string',
 765         format => 'urlencoded',
 766         format_description => 'model',
 767         maxLength => 40*3, # *3 since it's %xx url enoded
 768         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 769         optional => 1,
 770     },
 771 );
 772
 773 my %queues_fmt = (
 774     queues => {
 775         type => 'integer',
 776         format_description => 'nbqueues',
 777         description => "Number of queues.",
 778         minimum => 2,
 779         optional => 1
 780     }
 781 );
 782
 783 my $add_throttle_desc = sub {
 784     my ($key, $type, $what, $size, $longsize) = @_;
 785     $drivedesc_base{$key} = {
 786         type => $type,
 787         format_description => $size,
 788         description => "Maximum $what speed in $longsize per second.",
 789         optional => 1,
 790     };
 791 };
 792 # throughput: (leaky bucket)
 793 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 794 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 795 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 796 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 797 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 798 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 799 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 800 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 801 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 802
 803 # pools: (pool of IO before throttling starts taking effect)
 804 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 805 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 806 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 807 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 808 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 809 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 810
 811 my $ide_fmt = {
 812     %drivedesc_base,
 813     %rerror_fmt,
 814     %model_fmt,
 815 };
 816
 817 my $idedesc = {
 818     optional => 1,
 819     type => 'string', format => $ide_fmt,
 820     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 821 };
 822 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 823
 824 my $scsi_fmt = {
 825     %drivedesc_base,
 826     %iothread_fmt,
 827     %queues_fmt,
 828 };
 829 my $scsidesc = {
 830     optional => 1,
 831     type => 'string', format => $scsi_fmt,
 832     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 833 };
 834 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 835
 836 my $sata_fmt = {
 837     %drivedesc_base,
 838     %rerror_fmt,
 839 };
 840 my $satadesc = {
 841     optional => 1,
 842     type => 'string', format => $sata_fmt,
 843     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 844 };
 845 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 846
 847 my $virtio_fmt = {
 848     %drivedesc_base,
 849     %iothread_fmt,
 850     %rerror_fmt,
 851 };
 852 my $virtiodesc = {
 853     optional => 1,
 854     type => 'string', format => $virtio_fmt,
 855     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 856 };
 857 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 858
 859 my $alldrive_fmt = {
 860     %drivedesc_base,
 861     %rerror_fmt,
 862     %iothread_fmt,
 863     %model_fmt,
 864     %queues_fmt,
 865 };
 866
 867 my $usb_fmt = {
 868     host => {
 869         default_key => 1,
 870         type => 'string', format => 'pve-qm-usb-device',
 871         format_description => 'HOSTUSBDEVICE|spice',
 872         description => 'The Host USB device or port or the value spice',
 873     },
 874     usb3 => {
 875         optional => 1,
 876         type => 'boolean',
 877         format_description => 'yes|no',
 878         description => 'Specifies whether if given host option is a USB3 device or port',
 879     },
 880 };
 881
 882 my $usbdesc = {
 883     optional => 1,
 884     type => 'string', format => $usb_fmt,
 885     description => <<EODESCR,
 886 Configure an USB device (n is 0 to 4). This can be used to
 887 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 888
 889 'bus-port(.port)*' (decimal numbers) or
 890 'vendor_id:product_id' (hexadeciaml numbers) or
 891 'spice'
 892
 893 You can use the 'lsusb -t' command to list existing usb devices.
 894
 895 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 896
 897 The value 'spice' can be used to add a usb redirection devices for spice.
 898
 899 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 900
 901 EODESCR
 902 };
 903 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 904
 905 # NOTE: the match-groups of this regex are used in parse_hostpci
 906 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 907 my $hostpci_fmt = {
 908     host => {
 909         default_key => 1,
 910         type => 'string',
 911         pattern => qr/$PCIRE(;$PCIRE)*/,
 912         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 913         description => "The PCI ID of a host's PCI device or a list of PCI virtual functions of the host.",
 914     },
 915     rombar => {
 916         type => 'boolean',
 917         optional => 1,
 918         default => 1,
 919     },
 920     pcie => {
 921         type => 'boolean',
 922         optional => 1,
 923         default => 0,
 924     },
 925     'x-vga' => {
 926         type => 'boolean',
 927         optional => 1,
 928         default => 0,
 929     },
 930 };
 931 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
 932
 933 my $hostpcidesc = {
 934         optional => 1,
 935         type => 'string', format => 'pve-qm-hostpci',
 936         description => <<EODESCR,
 937 Map host pci devices. HOSTPCIDEVICE syntax is:
 938
 939 'bus:dev.func' (hexadecimal numbers)
 940
 941 You can us the 'lspci' command to list existing pci devices.
 942
 943 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 944
 945 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 946
 947 Experimental: user reported problems with this option.
 948 EODESCR
 949 };
 950 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 951
 952 my $serialdesc = {
 953         optional => 1,
 954         type => 'string',
 955         pattern => '(/dev/.+|socket)',
 956         description =>  <<EODESCR,
 957 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 958
 959 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 960
 961 Experimental: user reported problems with this option.
 962 EODESCR
 963 };
 964
 965 my $paralleldesc= {
 966         optional => 1,
 967         type => 'string',
 968         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 969         description =>  <<EODESCR,
 970 Map host parallel devices (n is 0 to 2).
 971
 972 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 973
 974 Experimental: user reported problems with this option.
 975 EODESCR
 976 };
 977
 978 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 979     $confdesc->{"parallel$i"} = $paralleldesc;
 980 }
 981
 982 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 983     $confdesc->{"serial$i"} = $serialdesc;
 984 }
 985
 986 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 987     $confdesc->{"hostpci$i"} = $hostpcidesc;
 988 }
 989
 990 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 991     $drivename_hash->{"ide$i"} = 1;
 992     $confdesc->{"ide$i"} = $idedesc;
 993 }
 994
 995 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 996     $drivename_hash->{"sata$i"} = 1;
 997     $confdesc->{"sata$i"} = $satadesc;
 998 }
 999
1000 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1001     $drivename_hash->{"scsi$i"} = 1;
1002     $confdesc->{"scsi$i"} = $scsidesc ;
1003 }
1004
1005 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1006     $drivename_hash->{"virtio$i"} = 1;
1007     $confdesc->{"virtio$i"} = $virtiodesc;
1008 }
1009
1010 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1011     $confdesc->{"usb$i"} = $usbdesc;
1012 }
1013
1014 my $unuseddesc = {
1015     optional => 1,
1016     type => 'string', format => 'pve-volume-id',
1017     description => "Reference to unused volumes.",
1018 };
1019
1020 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1021     $confdesc->{"unused$i"} = $unuseddesc;
1022 }
1023
1024 my $kvm_api_version = 0;
1025
1026 sub kvm_version {
1027
1028     return $kvm_api_version if $kvm_api_version;
1029
1030     my $fh = IO::File->new("</dev/kvm") ||
1031         return 0;
1032
1033     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1034         $kvm_api_version = $v;
1035     }
1036
1037     $fh->close();
1038
1039     return  $kvm_api_version;
1040 }
1041
1042 my $kvm_user_version;
1043
1044 sub kvm_user_version {
1045
1046     return $kvm_user_version if $kvm_user_version;
1047
1048     $kvm_user_version = 'unknown';
1049
1050     my $code = sub {
1051         my $line = shift;
1052         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1053             $kvm_user_version = $2;
1054         }
1055     };
1056
1057     eval { run_command("kvm -version", outfunc => $code); };
1058     warn $@ if $@;
1059
1060     return $kvm_user_version;
1061
1062 }
1063
1064 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1065
1066 sub valid_drive_names {
1067     # order is important - used to autoselect boot disk
1068     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1069             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1070             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1071             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1072 }
1073
1074 sub is_valid_drivename {
1075     my $dev = shift;
1076
1077     return defined($drivename_hash->{$dev});
1078 }
1079
1080 sub option_exists {
1081     my $key = shift;
1082     return defined($confdesc->{$key});
1083 }
1084
1085 sub nic_models {
1086     return $nic_model_list;
1087 }
1088
1089 sub os_list_description {
1090
1091     return {
1092         other => 'Other',
1093         wxp => 'Windows XP',
1094         w2k => 'Windows 2000',
1095         w2k3 =>, 'Windows 2003',
1096         w2k8 => 'Windows 2008',
1097         wvista => 'Windows Vista',
1098         win7 => 'Windows 7',
1099         win8 => 'Windows 8/2012',
1100         l24 => 'Linux 2.4',
1101         l26 => 'Linux 2.6',
1102     };
1103 }
1104
1105 my $cdrom_path;
1106
1107 sub get_cdrom_path {
1108
1109     return  $cdrom_path if $cdrom_path;
1110
1111     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1112     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1113     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1114 }
1115
1116 sub get_iso_path {
1117     my ($storecfg, $vmid, $cdrom) = @_;
1118
1119     if ($cdrom eq 'cdrom') {
1120         return get_cdrom_path();
1121     } elsif ($cdrom eq 'none') {
1122         return '';
1123     } elsif ($cdrom =~ m|^/|) {
1124         return $cdrom;
1125     } else {
1126         return PVE::Storage::path($storecfg, $cdrom);
1127     }
1128 }
1129
1130 # try to convert old style file names to volume IDs
1131 sub filename_to_volume_id {
1132     my ($vmid, $file, $media) = @_;
1133
1134     if (!($file eq 'none' || $file eq 'cdrom' ||
1135           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1136
1137         return undef if $file =~ m|/|;
1138
1139         if ($media && $media eq 'cdrom') {
1140             $file = "local:iso/$file";
1141         } else {
1142             $file = "local:$vmid/$file";
1143         }
1144     }
1145
1146     return $file;
1147 }
1148
1149 sub verify_media_type {
1150     my ($opt, $vtype, $media) = @_;
1151
1152     return if !$media;
1153
1154     my $etype;
1155     if ($media eq 'disk') {
1156         $etype = 'images';
1157     } elsif ($media eq 'cdrom') {
1158         $etype = 'iso';
1159     } else {
1160         die "internal error";
1161     }
1162
1163     return if ($vtype eq $etype);
1164
1165     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1166 }
1167
1168 sub cleanup_drive_path {
1169     my ($opt, $storecfg, $drive) = @_;
1170
1171     # try to convert filesystem paths to volume IDs
1172
1173     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1174         ($drive->{file} !~ m|^/dev/.+|) &&
1175         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1176         ($drive->{file} !~ m/^\d+$/)) {
1177         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1178         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1179         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1180         verify_media_type($opt, $vtype, $drive->{media});
1181         $drive->{file} = $volid;
1182     }
1183
1184     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1185 }
1186
1187 sub parse_hotplug_features {
1188     my ($data) = @_;
1189
1190     my $res = {};
1191
1192     return $res if $data eq '0';
1193
1194     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1195
1196     foreach my $feature (PVE::Tools::split_list($data)) {
1197         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1198             $res->{$1} = 1;
1199         } else {
1200             warn "ignoring unknown hotplug feature '$feature'\n";
1201         }
1202     }
1203     return $res;
1204 }
1205
1206 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1207 sub pve_verify_hotplug_features {
1208     my ($value, $noerr) = @_;
1209
1210     return $value if parse_hotplug_features($value);
1211
1212     return undef if $noerr;
1213
1214     die "unable to parse hotplug option\n";
1215 }
1216
1217 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1218 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1219 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1220 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1221 #        [,iothread=on][,serial=serial][,model=model]
1222
1223 sub parse_drive {
1224     my ($key, $data) = @_;
1225
1226     my ($interface, $index);
1227
1228     if ($key =~ m/^([^\d]+)(\d+)$/) {
1229         $interface = $1;
1230         $index = $2;
1231     } else {
1232         return undef;
1233     }
1234
1235     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1236                                      : $confdesc->{$key}->{format};
1237     if (!$desc) {
1238         warn "invalid drive key: $key\n";
1239         return undef;
1240     }
1241     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1242     return undef if !$res;
1243     $res->{interface} = $interface;
1244     $res->{index} = $index;
1245
1246     my $error = 0;
1247     foreach my $opt (qw(bps bps_rd bps_wr)) {
1248         if (my $bps = defined(delete $res->{$opt})) {
1249             if (defined($res->{"m$opt"})) {
1250                 warn "both $opt and m$opt specified\n";
1251                 ++$error;
1252                 next;
1253             }
1254             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1255         }
1256     }
1257     return undef if $error;
1258
1259     return undef if $res->{mbps_rd} && $res->{mbps};
1260     return undef if $res->{mbps_wr} && $res->{mbps};
1261     return undef if $res->{iops_rd} && $res->{iops};
1262     return undef if $res->{iops_wr} && $res->{iops};
1263
1264     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1265         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1266         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1267         return undef if $res->{interface} eq 'virtio';
1268     }
1269
1270     if (my $size = $res->{size}) {
1271         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1272     }
1273
1274     return $res;
1275 }
1276
1277 sub print_drive {
1278     my ($vmid, $drive) = @_;
1279     my $data = { %$drive };
1280     delete $data->{$_} for qw(index interface);
1281     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1282 }
1283
1284 sub scsi_inquiry {
1285     my($fh, $noerr) = @_;
1286
1287     my $SG_IO = 0x2285;
1288     my $SG_GET_VERSION_NUM = 0x2282;
1289
1290     my $versionbuf = "\x00" x 8;
1291     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1292     if (!$ret) {
1293         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1294         return undef;
1295     }
1296     my $version = unpack("I", $versionbuf);
1297     if ($version < 30000) {
1298         die "scsi generic interface too old\n"  if !$noerr;
1299         return undef;
1300     }
1301
1302     my $buf = "\x00" x 36;
1303     my $sensebuf = "\x00" x 8;
1304     my $cmd = pack("C x3 C x1", 0x12, 36);
1305
1306     # see /usr/include/scsi/sg.h
1307     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1308
1309     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1310                       length($sensebuf), 0, length($buf), $buf,
1311                       $cmd, $sensebuf, 6000);
1312
1313     $ret = ioctl($fh, $SG_IO, $packet);
1314     if (!$ret) {
1315         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1316         return undef;
1317     }
1318
1319     my @res = unpack($sg_io_hdr_t, $packet);
1320     if ($res[17] || $res[18]) {
1321         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1322         return undef;
1323     }
1324
1325     my $res = {};
1326     (my $byte0, my $byte1, $res->{vendor},
1327      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1328
1329     $res->{removable} = $byte1 & 128 ? 1 : 0;
1330     $res->{type} = $byte0 & 31;
1331
1332     return $res;
1333 }
1334
1335 sub path_is_scsi {
1336     my ($path) = @_;
1337
1338     my $fh = IO::File->new("+<$path") || return undef;
1339     my $res = scsi_inquiry($fh, 1);
1340     close($fh);
1341
1342     return $res;
1343 }
1344
1345 sub machine_type_is_q35 {
1346     my ($conf) = @_;
1347
1348     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1349 }
1350
1351 sub print_tabletdevice_full {
1352     my ($conf) = @_;
1353
1354     my $q35 = machine_type_is_q35($conf);
1355
1356     # we use uhci for old VMs because tablet driver was buggy in older qemu
1357     my $usbbus = $q35 ? "ehci" : "uhci";
1358
1359     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1360 }
1361
1362 sub print_drivedevice_full {
1363     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1364
1365     my $device = '';
1366     my $maxdev = 0;
1367
1368     if ($drive->{interface} eq 'virtio') {
1369         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1370         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1371         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1372     } elsif ($drive->{interface} eq 'scsi') {
1373
1374         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1375         my $unit = $drive->{index} % $maxdev;
1376         my $devicetype = 'hd';
1377         my $path = '';
1378         if (drive_is_cdrom($drive)) {
1379             $devicetype = 'cd';
1380         } else {
1381             if ($drive->{file} =~ m|^/|) {
1382                 $path = $drive->{file};
1383                 if (my $info = path_is_scsi($path)) {
1384                     if ($info->{type} == 0) {
1385                         $devicetype = 'block';
1386                     } elsif ($info->{type} == 1) { # tape
1387                         $devicetype = 'generic';
1388                     }
1389                 }
1390             } else {
1391                  $path = PVE::Storage::path($storecfg, $drive->{file});
1392             }
1393
1394             if($path =~ m/^iscsi\:\/\//){
1395                 $devicetype = 'generic';
1396             }
1397         }
1398
1399         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1400             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1401         } else {
1402             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1403         }
1404
1405     } elsif ($drive->{interface} eq 'ide'){
1406         $maxdev = 2;
1407         my $controller = int($drive->{index} / $maxdev);
1408         my $unit = $drive->{index} % $maxdev;
1409         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1410
1411         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1412         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1413             $model = URI::Escape::uri_unescape($model);
1414             $device .= ",model=$model";
1415         }
1416     } elsif ($drive->{interface} eq 'sata'){
1417         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1418         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1419         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1420     } elsif ($drive->{interface} eq 'usb') {
1421         die "implement me";
1422         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1423     } else {
1424         die "unsupported interface type";
1425     }
1426
1427     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1428
1429     return $device;
1430 }
1431
1432 sub get_initiator_name {
1433     my $initiator;
1434
1435     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1436     while (defined(my $line = <$fh>)) {
1437         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1438         $initiator = $1;
1439         last;
1440     }
1441     $fh->close();
1442
1443     return $initiator;
1444 }
1445
1446 sub print_drive_full {
1447     my ($storecfg, $vmid, $drive) = @_;
1448
1449     my $path;
1450     my $volid = $drive->{file};
1451     my $format;
1452
1453     if (drive_is_cdrom($drive)) {
1454         $path = get_iso_path($storecfg, $vmid, $volid);
1455     } else {
1456         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1457         if ($storeid) {
1458             $path = PVE::Storage::path($storecfg, $volid);
1459             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1460             $format = qemu_img_format($scfg, $volname);
1461         } else {
1462             $path = $volid;
1463             $format = "raw";
1464         }
1465    }
1466
1467     my $opts = '';
1468     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1469     foreach my $o (@qemu_drive_options) {
1470         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1471     }
1472     if (my $serial = $drive->{serial}) {
1473         $serial = URI::Escape::uri_unescape($serial);
1474         $opts .= ",serial=$serial";
1475     }
1476
1477     $opts .= ",format=$format" if $format && !$drive->{format};
1478
1479     foreach my $o (qw(bps bps_rd bps_wr)) {
1480         my $v = $drive->{"m$o"};
1481         $opts .= ",$o=" . int($v*1024*1024) if $v;
1482     }
1483
1484     my $cache_direct = 0;
1485
1486     if (my $cache = $drive->{cache}) {
1487         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1488     } elsif (!drive_is_cdrom($drive)) {
1489         $opts .= ",cache=none";
1490         $cache_direct = 1;
1491     }
1492
1493     # aio native works only with O_DIRECT
1494     if (!$drive->{aio}) {
1495         if($cache_direct) {
1496             $opts .= ",aio=native";
1497         } else {
1498             $opts .= ",aio=threads";
1499         }
1500     }
1501
1502     if (!drive_is_cdrom($drive)) {
1503         my $detectzeroes;
1504         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1505             $detectzeroes = 'off';
1506         } elsif ($drive->{discard}) {
1507             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1508         } else {
1509             # This used to be our default with discard not being specified:
1510             $detectzeroes = 'on';
1511         }
1512         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1513     }
1514
1515     my $pathinfo = $path ? "file=$path," : '';
1516
1517     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1518 }
1519
1520 sub print_netdevice_full {
1521     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1522
1523     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1524
1525     my $device = $net->{model};
1526     if ($net->{model} eq 'virtio') {
1527          $device = 'virtio-net-pci';
1528      };
1529
1530     my $pciaddr = print_pci_addr("$netid", $bridges);
1531     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1532     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1533         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1534         my $vectors = $net->{queues} * 2 + 2;
1535         $tmpstr .= ",vectors=$vectors,mq=on";
1536     }
1537     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1538
1539     if ($use_old_bios_files) {
1540         my $romfile;
1541         if ($device eq 'virtio-net-pci') {
1542             $romfile = 'pxe-virtio.rom';
1543         } elsif ($device eq 'e1000') {
1544             $romfile = 'pxe-e1000.rom';
1545         } elsif ($device eq 'ne2k') {
1546             $romfile = 'pxe-ne2k_pci.rom';
1547         } elsif ($device eq 'pcnet') {
1548             $romfile = 'pxe-pcnet.rom';
1549         } elsif ($device eq 'rtl8139') {
1550             $romfile = 'pxe-rtl8139.rom';
1551         }
1552         $tmpstr .= ",romfile=$romfile" if $romfile;
1553     }
1554
1555     return $tmpstr;
1556 }
1557
1558 sub print_netdev_full {
1559     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1560
1561     my $i = '';
1562     if ($netid =~ m/^net(\d+)$/) {
1563         $i = int($1);
1564     }
1565
1566     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1567
1568     my $ifname = "tap${vmid}i$i";
1569
1570     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1571     die "interface name '$ifname' is too long (max 15 character)\n"
1572         if length($ifname) >= 16;
1573
1574     my $vhostparam = '';
1575     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1576
1577     my $vmname = $conf->{name} || "vm$vmid";
1578
1579     my $netdev = "";
1580     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1581
1582     if ($net->{bridge}) {
1583         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1584     } else {
1585         $netdev = "type=user,id=$netid,hostname=$vmname";
1586     }
1587
1588     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1589
1590     return $netdev;
1591 }
1592
1593 sub drive_is_cdrom {
1594     my ($drive) = @_;
1595
1596     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1597
1598 }
1599
1600 sub parse_number_sets {
1601     my ($set) = @_;
1602     my $res = [];
1603     foreach my $part (split(/;/, $set)) {
1604         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1605             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1606             push @$res, [ $1, $2 ];
1607         } else {
1608             die "invalid range: $part\n";
1609         }
1610     }
1611     return $res;
1612 }
1613
1614 sub parse_numa {
1615     my ($data) = @_;
1616
1617     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1618     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1619     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1620     return $res;
1621 }
1622
1623 sub parse_hostpci {
1624     my ($value) = @_;
1625
1626     return undef if !$value;
1627
1628     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1629
1630     my @idlist = split(/;/, $res->{host});
1631     delete $res->{host};
1632     foreach my $id (@idlist) {
1633         if ($id =~ /^$PCIRE$/) {
1634             push @{$res->{pciid}}, { id => $1, function => ($2//'0') };
1635         } else {
1636             # should have been caught by parse_property_string already
1637             die "failed to parse PCI id: $id\n";
1638         }
1639     }
1640     return $res;
1641 }
1642
1643 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1644 sub parse_net {
1645     my ($data) = @_;
1646
1647     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1648     if ($@) {
1649         warn $@;
1650         return undef;
1651     }
1652     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1653     return $res;
1654 }
1655
1656 sub print_net {
1657     my $net = shift;
1658
1659     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1660 }
1661
1662 sub add_random_macs {
1663     my ($settings) = @_;
1664
1665     foreach my $opt (keys %$settings) {
1666         next if $opt !~ m/^net(\d+)$/;
1667         my $net = parse_net($settings->{$opt});
1668         next if !$net;
1669         $settings->{$opt} = print_net($net);
1670     }
1671 }
1672
1673 sub vm_is_volid_owner {
1674     my ($storecfg, $vmid, $volid) = @_;
1675
1676     if ($volid !~  m|^/|) {
1677         my ($path, $owner);
1678         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1679         if ($owner && ($owner == $vmid)) {
1680             return 1;
1681         }
1682     }
1683
1684     return undef;
1685 }
1686
1687 sub split_flagged_list {
1688     my $text = shift || '';
1689     $text =~ s/[,;]/ /g;
1690     $text =~ s/^\s+//;
1691     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1692 }
1693
1694 sub join_flagged_list {
1695     my ($how, $lst) = @_;
1696     join $how, map { $lst->{$_} . $_ } keys %$lst;
1697 }
1698
1699 sub vmconfig_delete_pending_option {
1700     my ($conf, $key, $force) = @_;
1701
1702     delete $conf->{pending}->{$key};
1703     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1704     $pending_delete_hash->{$key} = $force ? '!' : '';
1705     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1706 }
1707
1708 sub vmconfig_undelete_pending_option {
1709     my ($conf, $key) = @_;
1710
1711     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1712     delete $pending_delete_hash->{$key};
1713
1714     if (%$pending_delete_hash) {
1715         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1716     } else {
1717         delete $conf->{pending}->{delete};
1718     }
1719 }
1720
1721 sub vmconfig_register_unused_drive {
1722     my ($storecfg, $vmid, $conf, $drive) = @_;
1723
1724     if (!drive_is_cdrom($drive)) {
1725         my $volid = $drive->{file};
1726         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1727             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1728         }
1729     }
1730 }
1731
1732 sub vmconfig_cleanup_pending {
1733     my ($conf) = @_;
1734
1735     # remove pending changes when nothing changed
1736     my $changes;
1737     foreach my $opt (keys %{$conf->{pending}}) {
1738         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1739             $changes = 1;
1740             delete $conf->{pending}->{$opt};
1741         }
1742     }
1743
1744     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1745     my $pending_delete_hash = {};
1746     while (my ($opt, $force) = each %$current_delete_hash) {
1747         if (defined($conf->{$opt})) {
1748             $pending_delete_hash->{$opt} = $force;
1749         } else {
1750             $changes = 1;
1751         }
1752     }
1753
1754     if (%$pending_delete_hash) {
1755         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1756     } else {
1757         delete $conf->{pending}->{delete};
1758     }
1759
1760     return $changes;
1761 }
1762
1763 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1764 my $smbios1_fmt = {
1765     uuid => {
1766         type => 'string',
1767         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1768         format_description => 'UUID',
1769         optional => 1,
1770     },
1771     version => {
1772         type => 'string',
1773         pattern => '\S+',
1774         format_description => 'str',
1775         optional => 1,
1776     },
1777     serial => {
1778         type => 'string',
1779         pattern => '\S+',
1780         format_description => 'str',
1781         optional => 1,
1782     },
1783     manufacturer => {
1784         type => 'string',
1785         pattern => '\S+',
1786         format_description => 'name',
1787         optional => 1,
1788     },
1789     product => {
1790         type => 'string',
1791         pattern => '\S+',
1792         format_description => 'name',
1793         optional => 1,
1794     },
1795     sku => {
1796         type => 'string',
1797         pattern => '\S+',
1798         format_description => 'str',
1799         optional => 1,
1800     },
1801     family => {
1802         type => 'string',
1803         pattern => '\S+',
1804         format_description => 'str',
1805         optional => 1,
1806     },
1807 };
1808
1809 sub parse_smbios1 {
1810     my ($data) = @_;
1811
1812     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1813     warn $@ if $@;
1814     return $res;
1815 }
1816
1817 sub print_smbios1 {
1818     my ($smbios1) = @_;
1819     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1820 }
1821
1822 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1823
1824 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1825 sub verify_bootdisk {
1826     my ($value, $noerr) = @_;
1827
1828     return $value if is_valid_drivename($value);
1829
1830     return undef if $noerr;
1831
1832     die "invalid boot disk '$value'\n";
1833 }
1834
1835 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1836 sub verify_net {
1837     my ($value, $noerr) = @_;
1838
1839     return $value if parse_net($value);
1840
1841     return undef if $noerr;
1842
1843     die "unable to parse network options\n";
1844 }
1845
1846 sub parse_watchdog {
1847     my ($value) = @_;
1848
1849     return undef if !$value;
1850
1851     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1852     warn $@ if $@;
1853     return $res;
1854 }
1855
1856 sub parse_usb_device {
1857     my ($value) = @_;
1858
1859     return undef if !$value;
1860
1861     my $res = {};
1862     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1863         $res->{vendorid} = $2;
1864         $res->{productid} = $4;
1865     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1866         $res->{hostbus} = $1;
1867         $res->{hostport} = $2;
1868     } elsif ($value =~ m/^spice$/i) {
1869         $res->{spice} = 1;
1870     } else {
1871         return undef;
1872     }
1873
1874     return $res;
1875 }
1876
1877 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1878 sub verify_usb_device {
1879     my ($value, $noerr) = @_;
1880
1881     return $value if parse_usb_device($value);
1882
1883     return undef if $noerr;
1884
1885     die "unable to parse usb device\n";
1886 }
1887
1888 # add JSON properties for create and set function
1889 sub json_config_properties {
1890     my $prop = shift;
1891
1892     foreach my $opt (keys %$confdesc) {
1893         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1894         $prop->{$opt} = $confdesc->{$opt};
1895     }
1896
1897     return $prop;
1898 }
1899
1900 sub check_type {
1901     my ($key, $value) = @_;
1902
1903     die "unknown setting '$key'\n" if !$confdesc->{$key};
1904
1905     my $type = $confdesc->{$key}->{type};
1906
1907     if (!defined($value)) {
1908         die "got undefined value\n";
1909     }
1910
1911     if ($value =~ m/[\n\r]/) {
1912         die "property contains a line feed\n";
1913     }
1914
1915     if ($type eq 'boolean') {
1916         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1917         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1918         die "type check ('boolean') failed - got '$value'\n";
1919     } elsif ($type eq 'integer') {
1920         return int($1) if $value =~ m/^(\d+)$/;
1921         die "type check ('integer') failed - got '$value'\n";
1922     } elsif ($type eq 'number') {
1923         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1924         die "type check ('number') failed - got '$value'\n";
1925     } elsif ($type eq 'string') {
1926         if (my $fmt = $confdesc->{$key}->{format}) {
1927             if ($fmt eq 'pve-qm-drive') {
1928                 # special case - we need to pass $key to parse_drive()
1929                 my $drive = parse_drive($key, $value);
1930                 return $value if $drive;
1931                 die "unable to parse drive options\n";
1932             }
1933             PVE::JSONSchema::check_format($fmt, $value);
1934             return $value;
1935         }
1936         $value =~ s/^\"(.*)\"$/$1/;
1937         return $value;
1938     } else {
1939         die "internal error"
1940     }
1941 }
1942
1943 sub check_iommu_support{
1944     #fixme : need to check IOMMU support
1945     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1946
1947     my $iommu=1;
1948     return $iommu;
1949
1950 }
1951
1952 sub touch_config {
1953     my ($vmid) = @_;
1954
1955     my $conf = PVE::QemuConfig->config_file($vmid);
1956     utime undef, undef, $conf;
1957 }
1958
1959 sub destroy_vm {
1960     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1961
1962     my $conffile = PVE::QemuConfig->config_file($vmid);
1963
1964     my $conf = PVE::QemuConfig->load_config($vmid);
1965
1966     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1967
1968     # only remove disks owned by this VM
1969     foreach_drive($conf, sub {
1970         my ($ds, $drive) = @_;
1971
1972         return if drive_is_cdrom($drive);
1973
1974         my $volid = $drive->{file};
1975
1976         return if !$volid || $volid =~ m|^/|;
1977
1978         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1979         return if !$path || !$owner || ($owner != $vmid);
1980
1981         PVE::Storage::vdisk_free($storecfg, $volid);
1982     });
1983
1984     if ($keep_empty_config) {
1985         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1986     } else {
1987         unlink $conffile;
1988     }
1989
1990     # also remove unused disk
1991     eval {
1992         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1993
1994         eval {
1995             PVE::Storage::foreach_volid($dl, sub {
1996                 my ($volid, $sid, $volname, $d) = @_;
1997                 PVE::Storage::vdisk_free($storecfg, $volid);
1998             });
1999         };
2000         warn $@ if $@;
2001
2002     };
2003     warn $@ if $@;
2004 }
2005
2006 sub parse_vm_config {
2007     my ($filename, $raw) = @_;
2008
2009     return undef if !defined($raw);
2010
2011     my $res = {
2012         digest => Digest::SHA::sha1_hex($raw),
2013         snapshots => {},
2014         pending => {},
2015     };
2016
2017     $filename =~ m|/qemu-server/(\d+)\.conf$|
2018         || die "got strange filename '$filename'";
2019
2020     my $vmid = $1;
2021
2022     my $conf = $res;
2023     my $descr;
2024     my $section = '';
2025
2026     my @lines = split(/\n/, $raw);
2027     foreach my $line (@lines) {
2028         next if $line =~ m/^\s*$/;
2029
2030         if ($line =~ m/^\[PENDING\]\s*$/i) {
2031             $section = 'pending';
2032             if (defined($descr)) {
2033                 $descr =~ s/\s+$//;
2034                 $conf->{description} = $descr;
2035             }
2036             $descr = undef;
2037             $conf = $res->{$section} = {};
2038             next;
2039
2040         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2041             $section = $1;
2042             if (defined($descr)) {
2043                 $descr =~ s/\s+$//;
2044                 $conf->{description} = $descr;
2045             }
2046             $descr = undef;
2047             $conf = $res->{snapshots}->{$section} = {};
2048             next;
2049         }
2050
2051         if ($line =~ m/^\#(.*)\s*$/) {
2052             $descr = '' if !defined($descr);
2053             $descr .= PVE::Tools::decode_text($1) . "\n";
2054             next;
2055         }
2056
2057         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2058             $descr = '' if !defined($descr);
2059             $descr .= PVE::Tools::decode_text($2);
2060         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2061             $conf->{snapstate} = $1;
2062         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2063             my $key = $1;
2064             my $value = $2;
2065             $conf->{$key} = $value;
2066         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2067             my $value = $1;
2068             if ($section eq 'pending') {
2069                 $conf->{delete} = $value; # we parse this later
2070             } else {
2071                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2072             }
2073         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2074             my $key = $1;
2075             my $value = $2;
2076             eval { $value = check_type($key, $value); };
2077             if ($@) {
2078                 warn "vm $vmid - unable to parse value of '$key' - $@";
2079             } else {
2080                 my $fmt = $confdesc->{$key}->{format};
2081                 if ($fmt && $fmt eq 'pve-qm-drive') {
2082                     my $v = parse_drive($key, $value);
2083                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2084                         $v->{file} = $volid;
2085                         $value = print_drive($vmid, $v);
2086                     } else {
2087                         warn "vm $vmid - unable to parse value of '$key'\n";
2088                         next;
2089                     }
2090                 }
2091
2092                 if ($key eq 'cdrom') {
2093                     $conf->{ide2} = $value;
2094                 } else {
2095                     $conf->{$key} = $value;
2096                 }
2097             }
2098         }
2099     }
2100
2101     if (defined($descr)) {
2102         $descr =~ s/\s+$//;
2103         $conf->{description} = $descr;
2104     }
2105     delete $res->{snapstate}; # just to be sure
2106
2107     return $res;
2108 }
2109
2110 sub write_vm_config {
2111     my ($filename, $conf) = @_;
2112
2113     delete $conf->{snapstate}; # just to be sure
2114
2115     if ($conf->{cdrom}) {
2116         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2117         $conf->{ide2} = $conf->{cdrom};
2118         delete $conf->{cdrom};
2119     }
2120
2121     # we do not use 'smp' any longer
2122     if ($conf->{sockets}) {
2123         delete $conf->{smp};
2124     } elsif ($conf->{smp}) {
2125         $conf->{sockets} = $conf->{smp};
2126         delete $conf->{cores};
2127         delete $conf->{smp};
2128     }
2129
2130     my $used_volids = {};
2131
2132     my $cleanup_config = sub {
2133         my ($cref, $pending, $snapname) = @_;
2134
2135         foreach my $key (keys %$cref) {
2136             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2137                 $key eq 'snapstate' || $key eq 'pending';
2138             my $value = $cref->{$key};
2139             if ($key eq 'delete') {
2140                 die "propertry 'delete' is only allowed in [PENDING]\n"
2141                     if !$pending;
2142                 # fixme: check syntax?
2143                 next;
2144             }
2145             eval { $value = check_type($key, $value); };
2146             die "unable to parse value of '$key' - $@" if $@;
2147
2148             $cref->{$key} = $value;
2149
2150             if (!$snapname && is_valid_drivename($key)) {
2151                 my $drive = parse_drive($key, $value);
2152                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2153             }
2154         }
2155     };
2156
2157     &$cleanup_config($conf);
2158
2159     &$cleanup_config($conf->{pending}, 1);
2160
2161     foreach my $snapname (keys %{$conf->{snapshots}}) {
2162         die "internal error" if $snapname eq 'pending';
2163         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2164     }
2165
2166     # remove 'unusedX' settings if we re-add a volume
2167     foreach my $key (keys %$conf) {
2168         my $value = $conf->{$key};
2169         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2170             delete $conf->{$key};
2171         }
2172     }
2173
2174     my $generate_raw_config = sub {
2175         my ($conf, $pending) = @_;
2176
2177         my $raw = '';
2178
2179         # add description as comment to top of file
2180         if (defined(my $descr = $conf->{description})) {
2181             if ($descr) {
2182                 foreach my $cl (split(/\n/, $descr)) {
2183                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2184                 }
2185             } else {
2186                 $raw .= "#\n" if $pending;
2187             }
2188         }
2189
2190         foreach my $key (sort keys %$conf) {
2191             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2192             $raw .= "$key: $conf->{$key}\n";
2193         }
2194         return $raw;
2195     };
2196
2197     my $raw = &$generate_raw_config($conf);
2198
2199     if (scalar(keys %{$conf->{pending}})){
2200         $raw .= "\n[PENDING]\n";
2201         $raw .= &$generate_raw_config($conf->{pending}, 1);
2202     }
2203
2204     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2205         $raw .= "\n[$snapname]\n";
2206         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2207     }
2208
2209     return $raw;
2210 }
2211
2212 sub load_defaults {
2213
2214     my $res = {};
2215
2216     # we use static defaults from our JSON schema configuration
2217     foreach my $key (keys %$confdesc) {
2218         if (defined(my $default = $confdesc->{$key}->{default})) {
2219             $res->{$key} = $default;
2220         }
2221     }
2222
2223     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2224     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2225
2226     return $res;
2227 }
2228
2229 sub config_list {
2230     my $vmlist = PVE::Cluster::get_vmlist();
2231     my $res = {};
2232     return $res if !$vmlist || !$vmlist->{ids};
2233     my $ids = $vmlist->{ids};
2234
2235     foreach my $vmid (keys %$ids) {
2236         my $d = $ids->{$vmid};
2237         next if !$d->{node} || $d->{node} ne $nodename;
2238         next if !$d->{type} || $d->{type} ne 'qemu';
2239         $res->{$vmid}->{exists} = 1;
2240     }
2241     return $res;
2242 }
2243
2244 # test if VM uses local resources (to prevent migration)
2245 sub check_local_resources {
2246     my ($conf, $noerr) = @_;
2247
2248     my $loc_res = 0;
2249
2250     $loc_res = 1 if $conf->{hostusb}; # old syntax
2251     $loc_res = 1 if $conf->{hostpci}; # old syntax
2252
2253     foreach my $k (keys %$conf) {
2254         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2255         # sockets are safe: they will recreated be on the target side post-migrate
2256         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2257         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2258     }
2259
2260     die "VM uses local resources\n" if $loc_res && !$noerr;
2261
2262     return $loc_res;
2263 }
2264
2265 # check if used storages are available on all nodes (use by migrate)
2266 sub check_storage_availability {
2267     my ($storecfg, $conf, $node) = @_;
2268
2269     foreach_drive($conf, sub {
2270         my ($ds, $drive) = @_;
2271
2272         my $volid = $drive->{file};
2273         return if !$volid;
2274
2275         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2276         return if !$sid;
2277
2278         # check if storage is available on both nodes
2279         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2280         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2281    });
2282 }
2283
2284 # list nodes where all VM images are available (used by has_feature API)
2285 sub shared_nodes {
2286     my ($conf, $storecfg) = @_;
2287
2288     my $nodelist = PVE::Cluster::get_nodelist();
2289     my $nodehash = { map { $_ => 1 } @$nodelist };
2290     my $nodename = PVE::INotify::nodename();
2291
2292     foreach_drive($conf, sub {
2293         my ($ds, $drive) = @_;
2294
2295         my $volid = $drive->{file};
2296         return if !$volid;
2297
2298         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2299         if ($storeid) {
2300             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2301             if ($scfg->{disable}) {
2302                 $nodehash = {};
2303             } elsif (my $avail = $scfg->{nodes}) {
2304                 foreach my $node (keys %$nodehash) {
2305                     delete $nodehash->{$node} if !$avail->{$node};
2306                 }
2307             } elsif (!$scfg->{shared}) {
2308                 foreach my $node (keys %$nodehash) {
2309                     delete $nodehash->{$node} if $node ne $nodename
2310                 }
2311             }
2312         }
2313     });
2314
2315     return $nodehash
2316 }
2317
2318 sub check_cmdline {
2319     my ($pidfile, $pid) = @_;
2320
2321     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2322     if (defined($fh)) {
2323         my $line = <$fh>;
2324         $fh->close;
2325         return undef if !$line;
2326         my @param = split(/\0/, $line);
2327
2328         my $cmd = $param[0];
2329         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2330
2331         for (my $i = 0; $i < scalar (@param); $i++) {
2332             my $p = $param[$i];
2333             next if !$p;
2334             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2335                 my $p = $param[$i+1];
2336                 return 1 if $p && ($p eq $pidfile);
2337                 return undef;
2338             }
2339         }
2340     }
2341     return undef;
2342 }
2343
2344 sub check_running {
2345     my ($vmid, $nocheck, $node) = @_;
2346
2347     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2348
2349     die "unable to find configuration file for VM $vmid - no such machine\n"
2350         if !$nocheck && ! -f $filename;
2351
2352     my $pidfile = pidfile_name($vmid);
2353
2354     if (my $fd = IO::File->new("<$pidfile")) {
2355         my $st = stat($fd);
2356         my $line = <$fd>;
2357         close($fd);
2358
2359         my $mtime = $st->mtime;
2360         if ($mtime > time()) {
2361             warn "file '$filename' modified in future\n";
2362         }
2363
2364         if ($line =~ m/^(\d+)$/) {
2365             my $pid = $1;
2366             if (check_cmdline($pidfile, $pid)) {
2367                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2368                     return $pid;
2369                 }
2370             }
2371         }
2372     }
2373
2374     return undef;
2375 }
2376
2377 sub vzlist {
2378
2379     my $vzlist = config_list();
2380
2381     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2382
2383     while (defined(my $de = $fd->read)) {
2384         next if $de !~ m/^(\d+)\.pid$/;
2385         my $vmid = $1;
2386         next if !defined($vzlist->{$vmid});
2387         if (my $pid = check_running($vmid)) {
2388             $vzlist->{$vmid}->{pid} = $pid;
2389         }
2390     }
2391
2392     return $vzlist;
2393 }
2394
2395 sub disksize {
2396     my ($storecfg, $conf) = @_;
2397
2398     my $bootdisk = $conf->{bootdisk};
2399     return undef if !$bootdisk;
2400     return undef if !is_valid_drivename($bootdisk);
2401
2402     return undef if !$conf->{$bootdisk};
2403
2404     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2405     return undef if !defined($drive);
2406
2407     return undef if drive_is_cdrom($drive);
2408
2409     my $volid = $drive->{file};
2410     return undef if !$volid;
2411
2412     return $drive->{size};
2413 }
2414
2415 my $last_proc_pid_stat;
2416
2417 # get VM status information
2418 # This must be fast and should not block ($full == false)
2419 # We only query KVM using QMP if $full == true (this can be slow)
2420 sub vmstatus {
2421     my ($opt_vmid, $full) = @_;
2422
2423     my $res = {};
2424
2425     my $storecfg = PVE::Storage::config();
2426
2427     my $list = vzlist();
2428     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2429
2430     my $cpucount = $cpuinfo->{cpus} || 1;
2431
2432     foreach my $vmid (keys %$list) {
2433         next if $opt_vmid && ($vmid ne $opt_vmid);
2434
2435         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2436         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2437
2438         my $d = {};
2439         $d->{pid} = $list->{$vmid}->{pid};
2440
2441         # fixme: better status?
2442         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2443
2444         my $size = disksize($storecfg, $conf);
2445         if (defined($size)) {
2446             $d->{disk} = 0; # no info available
2447             $d->{maxdisk} = $size;
2448         } else {
2449             $d->{disk} = 0;
2450             $d->{maxdisk} = 0;
2451         }
2452
2453         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2454         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2455         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2456
2457         $d->{name} = $conf->{name} || "VM $vmid";
2458         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2459
2460         if ($conf->{balloon}) {
2461             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2462             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2463         }
2464
2465         $d->{uptime} = 0;
2466         $d->{cpu} = 0;
2467         $d->{mem} = 0;
2468
2469         $d->{netout} = 0;
2470         $d->{netin} = 0;
2471
2472         $d->{diskread} = 0;
2473         $d->{diskwrite} = 0;
2474
2475         $d->{template} = PVE::QemuConfig->is_template($conf);
2476
2477         $res->{$vmid} = $d;
2478     }
2479
2480     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2481     foreach my $dev (keys %$netdev) {
2482         next if $dev !~ m/^tap([1-9]\d*)i/;
2483         my $vmid = $1;
2484         my $d = $res->{$vmid};
2485         next if !$d;
2486
2487         $d->{netout} += $netdev->{$dev}->{receive};
2488         $d->{netin} += $netdev->{$dev}->{transmit};
2489
2490         if ($full) {
2491             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2492             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2493         }
2494
2495     }
2496
2497     my $ctime = gettimeofday;
2498
2499     foreach my $vmid (keys %$list) {
2500
2501         my $d = $res->{$vmid};
2502         my $pid = $d->{pid};
2503         next if !$pid;
2504
2505         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2506         next if !$pstat; # not running
2507
2508         my $used = $pstat->{utime} + $pstat->{stime};
2509
2510         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2511
2512         if ($pstat->{vsize}) {
2513             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2514         }
2515
2516         my $old = $last_proc_pid_stat->{$pid};
2517         if (!$old) {
2518             $last_proc_pid_stat->{$pid} = {
2519                 time => $ctime,
2520                 used => $used,
2521                 cpu => 0,
2522             };
2523             next;
2524         }
2525
2526         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2527
2528         if ($dtime > 1000) {
2529             my $dutime = $used -  $old->{used};
2530
2531             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2532             $last_proc_pid_stat->{$pid} = {
2533                 time => $ctime,
2534                 used => $used,
2535                 cpu => $d->{cpu},
2536             };
2537         } else {
2538             $d->{cpu} = $old->{cpu};
2539         }
2540     }
2541
2542     return $res if !$full;
2543
2544     my $qmpclient = PVE::QMPClient->new();
2545
2546     my $ballooncb = sub {
2547         my ($vmid, $resp) = @_;
2548
2549         my $info = $resp->{'return'};
2550         return if !$info->{max_mem};
2551
2552         my $d = $res->{$vmid};
2553
2554         # use memory assigned to VM
2555         $d->{maxmem} = $info->{max_mem};
2556         $d->{balloon} = $info->{actual};
2557
2558         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2559             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2560             $d->{freemem} = $info->{free_mem};
2561         }
2562
2563         $d->{ballooninfo} = $info;
2564     };
2565
2566     my $blockstatscb = sub {
2567         my ($vmid, $resp) = @_;
2568         my $data = $resp->{'return'} || [];
2569         my $totalrdbytes = 0;
2570         my $totalwrbytes = 0;
2571
2572         for my $blockstat (@$data) {
2573             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2574             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2575
2576             $blockstat->{device} =~ s/drive-//;
2577             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2578         }
2579         $res->{$vmid}->{diskread} = $totalrdbytes;
2580         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2581     };
2582
2583     my $statuscb = sub {
2584         my ($vmid, $resp) = @_;
2585
2586         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2587         # this fails if ballon driver is not loaded, so this must be
2588         # the last commnand (following command are aborted if this fails).
2589         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2590
2591         my $status = 'unknown';
2592         if (!defined($status = $resp->{'return'}->{status})) {
2593             warn "unable to get VM status\n";
2594             return;
2595         }
2596
2597         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2598     };
2599
2600     foreach my $vmid (keys %$list) {
2601         next if $opt_vmid && ($vmid ne $opt_vmid);
2602         next if !$res->{$vmid}->{pid}; # not running
2603         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2604     }
2605
2606     $qmpclient->queue_execute(undef, 1);
2607
2608     foreach my $vmid (keys %$list) {
2609         next if $opt_vmid && ($vmid ne $opt_vmid);
2610         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2611     }
2612
2613     return $res;
2614 }
2615
2616 sub foreach_dimm {
2617     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2618
2619     my $dimm_id = 0;
2620     my $current_size = 1024;
2621     my $dimm_size = 512;
2622     return if $current_size == $memory;
2623
2624     for (my $j = 0; $j < 8; $j++) {
2625         for (my $i = 0; $i < 32; $i++) {
2626             my $name = "dimm${dimm_id}";
2627             $dimm_id++;
2628             my $numanode = $i % $sockets;
2629             $current_size += $dimm_size;
2630             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2631             return  $current_size if $current_size >= $memory;
2632         }
2633         $dimm_size *= 2;
2634     }
2635 }
2636
2637 sub foreach_reverse_dimm {
2638     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2639
2640     my $dimm_id = 253;
2641     my $current_size = 4177920;
2642     my $dimm_size = 65536;
2643     return if $current_size == $memory;
2644
2645     for (my $j = 0; $j < 8; $j++) {
2646         for (my $i = 0; $i < 32; $i++) {
2647             my $name = "dimm${dimm_id}";
2648             $dimm_id--;
2649             my $numanode = $i % $sockets;
2650             $current_size -= $dimm_size;
2651             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2652             return  $current_size if $current_size <= $memory;
2653         }
2654         $dimm_size /= 2;
2655     }
2656 }
2657
2658 sub foreach_drive {
2659     my ($conf, $func) = @_;
2660
2661     foreach my $ds (valid_drive_names()) {
2662         next if !defined($conf->{$ds});
2663
2664         my $drive = parse_drive($ds, $conf->{$ds});
2665         next if !$drive;
2666
2667         &$func($ds, $drive);
2668     }
2669 }
2670
2671 sub foreach_volid {
2672     my ($conf, $func) = @_;
2673
2674     my $volhash = {};
2675
2676     my $test_volid = sub {
2677         my ($volid, $is_cdrom) = @_;
2678
2679         return if !$volid;
2680
2681         $volhash->{$volid} = $is_cdrom || 0;
2682     };
2683
2684     foreach_drive($conf, sub {
2685         my ($ds, $drive) = @_;
2686         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2687     });
2688
2689     foreach my $snapname (keys %{$conf->{snapshots}}) {
2690         my $snap = $conf->{snapshots}->{$snapname};
2691         &$test_volid($snap->{vmstate}, 0);
2692         foreach_drive($snap, sub {
2693             my ($ds, $drive) = @_;
2694             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2695         });
2696     }
2697
2698     foreach my $volid (keys %$volhash) {
2699         &$func($volid, $volhash->{$volid});
2700     }
2701 }
2702
2703 sub vga_conf_has_spice {
2704     my ($vga) = @_;
2705
2706     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2707
2708     return $1 || 1;
2709 }
2710
2711 sub config_to_command {
2712     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2713
2714     my $cmd = [];
2715     my $globalFlags = [];
2716     my $machineFlags = [];
2717     my $rtcFlags = [];
2718     my $cpuFlags = [];
2719     my $devices = [];
2720     my $pciaddr = '';
2721     my $bridges = {};
2722     my $kvmver = kvm_user_version();
2723     my $vernum = 0; # unknown
2724     my $ostype = $conf->{ostype};
2725     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2726         $vernum = $1*1000000+$2*1000;
2727     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2728         $vernum = $1*1000000+$2*1000+$3;
2729     }
2730
2731     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2732
2733     my $have_ovz = -f '/proc/vz/vestat';
2734
2735     my $q35 = machine_type_is_q35($conf);
2736     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2737     my $machine_type = $forcemachine || $conf->{machine};
2738     my $use_old_bios_files = undef;
2739     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2740
2741     my $cpuunits = defined($conf->{cpuunits}) ?
2742             $conf->{cpuunits} : $defaults->{cpuunits};
2743
2744     push @$cmd, '/usr/bin/systemd-run';
2745     push @$cmd, '--scope';
2746     push @$cmd, '--slice', "qemu";
2747     push @$cmd, '--unit', $vmid;
2748     # set KillMode=none, so that systemd don't kill those scopes
2749     # at shutdown (pve-manager service should stop the VMs instead)
2750     push @$cmd, '-p', "KillMode=none";
2751     push @$cmd, '-p', "CPUShares=$cpuunits";
2752     if ($conf->{cpulimit}) {
2753         my $cpulimit = int($conf->{cpulimit} * 100);
2754         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2755     }
2756
2757     push @$cmd, '/usr/bin/kvm';
2758
2759     push @$cmd, '-id', $vmid;
2760
2761     my $use_virtio = 0;
2762
2763     my $qmpsocket = qmp_socket($vmid);
2764     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2765     push @$cmd, '-mon', "chardev=qmp,mode=control";
2766
2767
2768     push @$cmd, '-pidfile' , pidfile_name($vmid);
2769
2770     push @$cmd, '-daemonize';
2771
2772     if ($conf->{smbios1}) {
2773         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2774     }
2775
2776     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2777         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2778         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2779         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2780         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2781         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2782         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2783     }
2784
2785     if ($q35) {
2786         # the q35 chipset support native usb2, so we enable usb controller
2787         # by default for this machine type
2788         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2789     } else {
2790         $pciaddr = print_pci_addr("piix3", $bridges);
2791         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2792
2793         my $use_usb2 = 0;
2794         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2795             next if !$conf->{"usb$i"};
2796             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2797             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2798             $use_usb2 = 1;
2799         }
2800         # include usb device config
2801         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2802     }
2803
2804     # add usb3 controller if needed
2805
2806     my $use_usb3 = 0;
2807     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2808         next if !$conf->{"usb$i"};
2809         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2810         next if !$d || !$d->{usb3};
2811         $use_usb3 = 1;
2812     }
2813
2814     $pciaddr = print_pci_addr("xhci", $bridges);
2815     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2816
2817     my $vga = $conf->{vga};
2818
2819     my $qxlnum = vga_conf_has_spice($vga);
2820     $vga = 'qxl' if $qxlnum;
2821
2822     if (!$vga) {
2823         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2824                                 $conf->{ostype} eq 'win7' ||
2825                                 $conf->{ostype} eq 'w2k8')) {
2826             $vga = 'std';
2827         } else {
2828             $vga = 'cirrus';
2829         }
2830     }
2831
2832     # enable absolute mouse coordinates (needed by vnc)
2833     my $tablet;
2834     if (defined($conf->{tablet})) {
2835         $tablet = $conf->{tablet};
2836     } else {
2837         $tablet = $defaults->{tablet};
2838         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2839         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2840     }
2841
2842     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2843
2844     my $kvm_off = 0;
2845     # host pci devices
2846     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2847         my $d = parse_hostpci($conf->{"hostpci$i"});
2848         next if !$d;
2849
2850         my $pcie = $d->{pcie};
2851         if($pcie){
2852             die "q35 machine model is not enabled" if !$q35;
2853             $pciaddr = print_pcie_addr("hostpci$i");
2854         }else{
2855             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2856         }
2857
2858         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2859         my $xvga = '';
2860         if ($d->{'x-vga'}) {
2861             $xvga = ',x-vga=on';
2862             $kvm_off = 1;
2863             $vga = 'none';
2864             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2865                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2866             }
2867             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2868                 $xvga = "";
2869             }
2870         }
2871         my $pcidevices = $d->{pciid};
2872         my $multifunction = 1 if @$pcidevices > 1;
2873
2874         my $j=0;
2875         foreach my $pcidevice (@$pcidevices) {
2876
2877             my $id = "hostpci$i";
2878             $id .= ".$j" if $multifunction;
2879             my $addr = $pciaddr;
2880             $addr .= ".$j" if $multifunction;
2881             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2882
2883             if($j == 0){
2884                 $devicestr .= "$rombar$xvga";
2885                 $devicestr .= ",multifunction=on" if $multifunction;
2886             }
2887
2888             push @$devices, '-device', $devicestr;
2889             $j++;
2890         }
2891     }
2892
2893     # usb devices
2894     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2895         next if !$conf->{"usb$i"};
2896         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2897         next if !$d;
2898
2899         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2900         my $usbbus = '';
2901         if (defined($d->{usb3}) && $d->{usb3}) {
2902             $usbbus = ',bus=xhci.0';
2903         }
2904
2905         if (defined($d->{host})) {
2906             $d = parse_usb_device($d->{host});
2907             if (defined($d->{vendorid}) && defined($d->{productid})) {
2908                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2909             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2910                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2911             } elsif (defined($d->{spice}) && $d->{spice}) {
2912                 # usb redir support for spice, currently no usb3
2913                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2914                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2915             }
2916         }
2917     }
2918
2919     # serial devices
2920     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2921         if (my $path = $conf->{"serial$i"}) {
2922             if ($path eq 'socket') {
2923                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2924                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2925                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2926             } else {
2927                 die "no such serial device\n" if ! -c $path;
2928                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2929                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2930             }
2931         }
2932     }
2933
2934     # parallel devices
2935     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2936         if (my $path = $conf->{"parallel$i"}) {
2937             die "no such parallel device\n" if ! -c $path;
2938             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2939             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2940             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2941         }
2942     }
2943
2944     my $vmname = $conf->{name} || "vm$vmid";
2945
2946     push @$cmd, '-name', $vmname;
2947
2948     my $sockets = 1;
2949     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2950     $sockets = $conf->{sockets} if  $conf->{sockets};
2951
2952     my $cores = $conf->{cores} || 1;
2953
2954     my $maxcpus = $sockets * $cores;
2955
2956     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2957
2958     my $allowed_vcpus = $cpuinfo->{cpus};
2959
2960     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2961         if ($allowed_vcpus < $maxcpus);
2962
2963     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2964
2965     push @$cmd, '-nodefaults';
2966
2967     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2968
2969     my $bootindex_hash = {};
2970     my $i = 1;
2971     foreach my $o (split(//, $bootorder)) {
2972         $bootindex_hash->{$o} = $i*100;
2973         $i++;
2974     }
2975
2976     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2977
2978     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2979
2980     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2981
2982     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2983
2984     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2985         my $socket = vnc_socket($vmid);
2986         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2987     } else {
2988         push @$cmd, '-nographic';
2989     }
2990
2991     # time drift fix
2992     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2993
2994     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2995     my $useLocaltime = $conf->{localtime};
2996
2997     if ($ostype) {
2998         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2999
3000         if ($ostype =~ m/^w/) { # windows
3001             $useLocaltime = 1 if !defined($conf->{localtime});
3002
3003             # use time drift fix when acpi is enabled
3004             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3005                 $tdf = 1 if !defined($conf->{tdf});
3006             }
3007         }
3008
3009         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3010             $ostype eq 'wvista') {
3011             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3012             push @$cmd, '-no-hpet';
3013             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3014                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3015                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3016                 push @$cpuFlags , 'hv_time' if !$nokvm;
3017
3018             } else {
3019                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3020             }
3021         }
3022
3023         if ($ostype eq 'win7' || $ostype eq 'win8') {
3024             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3025         }
3026     }
3027
3028     push @$rtcFlags, 'driftfix=slew' if $tdf;
3029
3030     if ($nokvm) {
3031         push @$machineFlags, 'accel=tcg';
3032     } else {
3033         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3034     }
3035
3036     if ($machine_type) {
3037         push @$machineFlags, "type=${machine_type}";
3038     }
3039
3040     if ($conf->{startdate}) {
3041         push @$rtcFlags, "base=$conf->{startdate}";
3042     } elsif ($useLocaltime) {
3043         push @$rtcFlags, 'base=localtime';
3044     }
3045
3046     my $cpu = $nokvm ? "qemu64" : "kvm64";
3047     if (my $cputype = $conf->{cpu}) {
3048         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3049             or die "Cannot parse cpu description: $cputype\n";
3050         $cpu = $cpuconf->{cputype};
3051         $kvm_off = 1 if $cpuconf->{hidden};
3052     }
3053
3054     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3055
3056     push @$cpuFlags , '-x2apic'
3057         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3058
3059     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3060
3061     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3062
3063     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3064
3065         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3066         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3067     }
3068
3069     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3070
3071     push @$cpuFlags, 'kvm=off' if $kvm_off;
3072
3073     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3074
3075     push @$cmd, '-cpu', $cpu;
3076
3077     my $memory = $conf->{memory} || $defaults->{memory};
3078     my $static_memory = 0;
3079     my $dimm_memory = 0;
3080
3081     if ($hotplug_features->{memory}) {
3082         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3083         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3084         $static_memory = $STATICMEM;
3085         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3086         $dimm_memory = $memory - $static_memory;
3087         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3088
3089     } else {
3090
3091         $static_memory = $memory;
3092         push @$cmd, '-m', $static_memory;
3093     }
3094
3095     if ($conf->{numa}) {
3096
3097         my $numa_totalmemory = undef;
3098         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3099             next if !$conf->{"numa$i"};
3100             my $numa = parse_numa($conf->{"numa$i"});
3101             next if !$numa;
3102             # memory
3103             die "missing numa node$i memory value\n" if !$numa->{memory};
3104             my $numa_memory = $numa->{memory};
3105             $numa_totalmemory += $numa_memory;
3106             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3107
3108             # cpus
3109             my $cpulists = $numa->{cpus};
3110             die "missing numa node$i cpus\n" if !defined($cpulists);
3111             my $cpus = join(',', map {
3112                 my ($start, $end) = @$_;
3113                 defined($end) ? "$start-$end" : $start
3114             } @$cpulists);
3115
3116             # hostnodes
3117             my $hostnodelists = $numa->{hostnodes};
3118             if (defined($hostnodelists)) {
3119                 my $hostnodes;
3120                 foreach my $hostnoderange (@$hostnodelists) {
3121                     my ($start, $end) = @$hostnoderange;
3122                     $hostnodes .= ',' if $hostnodes;
3123                     $hostnodes .= $start;
3124                     $hostnodes .= "-$end" if defined($end);
3125                     $end //= $start;
3126                     for (my $i = $start; $i <= $end; ++$i ) {
3127                         die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3128                     }
3129                 }
3130
3131                 # policy
3132                 my $policy = $numa->{policy};
3133                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3134                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3135             }
3136
3137             push @$cmd, '-object', $numa_object;
3138             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3139         }
3140
3141         die "total memory for NUMA nodes must be equal to vm static memory\n"
3142             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3143
3144         #if no custom tology, we split memory and cores across numa nodes
3145         if(!$numa_totalmemory) {
3146
3147             my $numa_memory = ($static_memory / $sockets) . "M";
3148
3149             for (my $i = 0; $i < $sockets; $i++)  {
3150
3151                 my $cpustart = ($cores * $i);
3152                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3153                 my $cpus = $cpustart;
3154                 $cpus .= "-$cpuend" if $cpuend;
3155
3156                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3157                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3158             }
3159         }
3160     }
3161
3162     if ($hotplug_features->{memory}) {
3163         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3164             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3165             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3166             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3167
3168             #if dimm_memory is not aligned to dimm map
3169             if($current_size > $memory) {
3170                  $conf->{memory} = $current_size;
3171                  PVE::QemuConfig->write_config($vmid, $conf);
3172             }
3173         });
3174     }
3175
3176     push @$cmd, '-S' if $conf->{freeze};
3177
3178     # set keyboard layout
3179     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3180     push @$cmd, '-k', $kb if $kb;
3181
3182     # enable sound
3183     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3184     #push @$cmd, '-soundhw', 'es1370';
3185     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3186
3187     if($conf->{agent}) {
3188         my $qgasocket = qmp_socket($vmid, 1);
3189         my $pciaddr = print_pci_addr("qga0", $bridges);
3190         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3191         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3192         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3193     }
3194
3195     my $spice_port;
3196
3197     if ($qxlnum) {
3198         if ($qxlnum > 1) {
3199             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3200                 for(my $i = 1; $i < $qxlnum; $i++){
3201                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3202                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3203                 }
3204             } else {
3205                 # assume other OS works like Linux
3206                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3207                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3208             }
3209         }
3210
3211         my $pciaddr = print_pci_addr("spice", $bridges);
3212
3213         my $nodename = PVE::INotify::nodename();
3214         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3215         $spice_port = PVE::Tools::next_spice_port($pfamily);
3216
3217         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3218
3219         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3220         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3221         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3222     }
3223
3224     # enable balloon by default, unless explicitly disabled
3225     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3226         $pciaddr = print_pci_addr("balloon0", $bridges);
3227         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3228     }
3229
3230     if ($conf->{watchdog}) {
3231         my $wdopts = parse_watchdog($conf->{watchdog});
3232         $pciaddr = print_pci_addr("watchdog", $bridges);
3233         my $watchdog = $wdopts->{model} || 'i6300esb';
3234         push @$devices, '-device', "$watchdog$pciaddr";
3235         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3236     }
3237
3238     my $vollist = [];
3239     my $scsicontroller = {};
3240     my $ahcicontroller = {};
3241     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3242
3243     # Add iscsi initiator name if available
3244     if (my $initiator = get_initiator_name()) {
3245         push @$devices, '-iscsi', "initiator-name=$initiator";
3246     }
3247
3248     foreach_drive($conf, sub {
3249         my ($ds, $drive) = @_;
3250
3251         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3252             push @$vollist, $drive->{file};
3253         }
3254
3255         $use_virtio = 1 if $ds =~ m/^virtio/;
3256
3257         if (drive_is_cdrom ($drive)) {
3258             if ($bootindex_hash->{d}) {
3259                 $drive->{bootindex} = $bootindex_hash->{d};
3260                 $bootindex_hash->{d} += 1;
3261             }
3262         } else {
3263             if ($bootindex_hash->{c}) {
3264                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3265                 $bootindex_hash->{c} += 1;
3266             }
3267         }
3268
3269         if($drive->{interface} eq 'virtio'){
3270            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3271         }
3272
3273         if ($drive->{interface} eq 'scsi') {
3274
3275             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3276
3277             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3278             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3279
3280             my $iothread = '';
3281             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3282                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3283                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3284             }
3285
3286             my $queues = '';
3287             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3288                 $queues = ",num_queues=$drive->{queues}";
3289             }
3290
3291             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3292             $scsicontroller->{$controller}=1;
3293         }
3294
3295         if ($drive->{interface} eq 'sata') {
3296            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3297            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3298            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3299            $ahcicontroller->{$controller}=1;
3300         }
3301
3302         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3303         push @$devices, '-drive',$drive_cmd;
3304         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3305     });
3306
3307     for (my $i = 0; $i < $MAX_NETS; $i++) {
3308          next if !$conf->{"net$i"};
3309          my $d = parse_net($conf->{"net$i"});
3310          next if !$d;
3311
3312          $use_virtio = 1 if $d->{model} eq 'virtio';
3313
3314          if ($bootindex_hash->{n}) {
3315             $d->{bootindex} = $bootindex_hash->{n};
3316             $bootindex_hash->{n} += 1;
3317          }
3318
3319          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3320          push @$devices, '-netdev', $netdevfull;
3321
3322          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3323          push @$devices, '-device', $netdevicefull;
3324     }
3325
3326     if (!$q35) {
3327         # add pci bridges
3328         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3329            $bridges->{1} = 1;
3330            $bridges->{2} = 1;
3331         }
3332
3333         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3334
3335         while (my ($k, $v) = each %$bridges) {
3336             $pciaddr = print_pci_addr("pci.$k");
3337             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3338         }
3339     }
3340
3341     # add custom args
3342     if ($conf->{args}) {
3343         my $aa = PVE::Tools::split_args($conf->{args});
3344         push @$cmd, @$aa;
3345     }
3346
3347     push @$cmd, @$devices;
3348     push @$cmd, '-rtc', join(',', @$rtcFlags)
3349         if scalar(@$rtcFlags);
3350     push @$cmd, '-machine', join(',', @$machineFlags)
3351         if scalar(@$machineFlags);
3352     push @$cmd, '-global', join(',', @$globalFlags)
3353         if scalar(@$globalFlags);
3354
3355     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3356 }
3357
3358 sub vnc_socket {
3359     my ($vmid) = @_;
3360     return "${var_run_tmpdir}/$vmid.vnc";
3361 }
3362
3363 sub spice_port {
3364     my ($vmid) = @_;
3365
3366     my $res = vm_mon_cmd($vmid, 'query-spice');
3367
3368     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3369 }
3370
3371 sub qmp_socket {
3372     my ($vmid, $qga) = @_;
3373     my $sockettype = $qga ? 'qga' : 'qmp';
3374     return "${var_run_tmpdir}/$vmid.$sockettype";
3375 }
3376
3377 sub pidfile_name {
3378     my ($vmid) = @_;
3379     return "${var_run_tmpdir}/$vmid.pid";
3380 }
3381
3382 sub vm_devices_list {
3383     my ($vmid) = @_;
3384
3385     my $res = vm_mon_cmd($vmid, 'query-pci');
3386     my $devices = {};
3387     foreach my $pcibus (@$res) {
3388         foreach my $device (@{$pcibus->{devices}}) {
3389             next if !$device->{'qdev_id'};
3390             if ($device->{'pci_bridge'}) {
3391                 $devices->{$device->{'qdev_id'}} = 1;
3392                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3393                     next if !$bridge_device->{'qdev_id'};
3394                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3395                     $devices->{$device->{'qdev_id'}}++;
3396                 }
3397             } else {
3398                 $devices->{$device->{'qdev_id'}} = 1;
3399             }
3400         }
3401     }
3402
3403     my $resblock = vm_mon_cmd($vmid, 'query-block');
3404     foreach my $block (@$resblock) {
3405         if($block->{device} =~ m/^drive-(\S+)/){
3406                 $devices->{$1} = 1;
3407         }
3408     }
3409
3410     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3411     foreach my $mice (@$resmice) {
3412         if ($mice->{name} eq 'QEMU HID Tablet') {
3413             $devices->{tablet} = 1;
3414             last;
3415         }
3416     }
3417
3418     return $devices;
3419 }
3420
3421 sub vm_deviceplug {
3422     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3423
3424     my $q35 = machine_type_is_q35($conf);
3425
3426     my $devices_list = vm_devices_list($vmid);
3427     return 1 if defined($devices_list->{$deviceid});
3428
3429     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3430
3431     if ($deviceid eq 'tablet') {
3432
3433         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3434
3435     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3436
3437         qemu_iothread_add($vmid, $deviceid, $device);
3438
3439         qemu_driveadd($storecfg, $vmid, $device);
3440         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3441
3442         qemu_deviceadd($vmid, $devicefull);
3443         eval { qemu_deviceaddverify($vmid, $deviceid); };
3444         if (my $err = $@) {
3445             eval { qemu_drivedel($vmid, $deviceid); };
3446             warn $@ if $@;
3447             die $err;
3448         }
3449
3450     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3451
3452
3453         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3454         my $pciaddr = print_pci_addr($deviceid);
3455         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3456
3457         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3458
3459         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3460             qemu_iothread_add($vmid, $deviceid, $device);
3461             $devicefull .= ",iothread=iothread-$deviceid";
3462         }
3463
3464         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3465             $devicefull .= ",num_queues=$device->{queues}";
3466         }
3467
3468         qemu_deviceadd($vmid, $devicefull);
3469         qemu_deviceaddverify($vmid, $deviceid);
3470
3471     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3472
3473         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3474         qemu_driveadd($storecfg, $vmid, $device);
3475
3476         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3477         eval { qemu_deviceadd($vmid, $devicefull); };
3478         if (my $err = $@) {
3479             eval { qemu_drivedel($vmid, $deviceid); };
3480             warn $@ if $@;
3481             die $err;
3482         }
3483
3484     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3485
3486         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3487
3488         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3489         my $use_old_bios_files = undef;
3490         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3491
3492         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3493         qemu_deviceadd($vmid, $netdevicefull);
3494         eval { qemu_deviceaddverify($vmid, $deviceid); };
3495         if (my $err = $@) {
3496             eval { qemu_netdevdel($vmid, $deviceid); };
3497             warn $@ if $@;
3498             die $err;
3499         }
3500
3501     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3502
3503         my $bridgeid = $2;
3504         my $pciaddr = print_pci_addr($deviceid);
3505         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3506
3507         qemu_deviceadd($vmid, $devicefull);
3508         qemu_deviceaddverify($vmid, $deviceid);
3509
3510     } else {
3511         die "can't hotplug device '$deviceid'\n";
3512     }
3513
3514     return 1;
3515 }
3516
3517 # fixme: this should raise exceptions on error!
3518 sub vm_deviceunplug {
3519     my ($vmid, $conf, $deviceid) = @_;
3520
3521     my $devices_list = vm_devices_list($vmid);
3522     return 1 if !defined($devices_list->{$deviceid});
3523
3524     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3525
3526     if ($deviceid eq 'tablet') {
3527
3528         qemu_devicedel($vmid, $deviceid);
3529
3530     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3531
3532         qemu_devicedel($vmid, $deviceid);
3533         qemu_devicedelverify($vmid, $deviceid);
3534         qemu_drivedel($vmid, $deviceid);
3535         qemu_iothread_del($conf, $vmid, $deviceid);
3536
3537     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3538
3539         qemu_devicedel($vmid, $deviceid);
3540         qemu_devicedelverify($vmid, $deviceid);
3541         qemu_iothread_del($conf, $vmid, $deviceid);
3542
3543     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3544
3545         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3546         my $device = parse_drive($deviceid, $conf->{$deviceid});
3547         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3548
3549         qemu_devicedel($vmid, $deviceid);
3550         qemu_drivedel($vmid, $deviceid);
3551         qemu_deletescsihw($conf, $vmid, $deviceid);
3552
3553     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3554
3555         qemu_devicedel($vmid, $deviceid);
3556         qemu_devicedelverify($vmid, $deviceid);
3557         qemu_netdevdel($vmid, $deviceid);
3558
3559     } else {
3560         die "can't unplug device '$deviceid'\n";
3561     }
3562
3563     return 1;
3564 }
3565
3566 sub qemu_deviceadd {
3567     my ($vmid, $devicefull) = @_;
3568
3569     $devicefull = "driver=".$devicefull;
3570     my %options =  split(/[=,]/, $devicefull);
3571
3572     vm_mon_cmd($vmid, "device_add" , %options);
3573 }
3574
3575 sub qemu_devicedel {
3576     my ($vmid, $deviceid) = @_;
3577
3578     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3579 }
3580
3581 sub qemu_iothread_add {
3582     my($vmid, $deviceid, $device) = @_;
3583
3584     if ($device->{iothread}) {
3585         my $iothreads = vm_iothreads_list($vmid);
3586         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3587     }
3588 }
3589
3590 sub qemu_iothread_del {
3591     my($conf, $vmid, $deviceid) = @_;
3592
3593     my $device = parse_drive($deviceid, $conf->{$deviceid});
3594     if ($device->{iothread}) {
3595         my $iothreads = vm_iothreads_list($vmid);
3596         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3597     }
3598 }
3599
3600 sub qemu_objectadd {
3601     my($vmid, $objectid, $qomtype) = @_;
3602
3603     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3604
3605     return 1;
3606 }
3607
3608 sub qemu_objectdel {
3609     my($vmid, $objectid) = @_;
3610
3611     vm_mon_cmd($vmid, "object-del", id => $objectid);
3612
3613     return 1;
3614 }
3615
3616 sub qemu_driveadd {
3617     my ($storecfg, $vmid, $device) = @_;
3618
3619     my $drive = print_drive_full($storecfg, $vmid, $device);
3620     $drive =~ s/\\/\\\\/g;
3621     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3622
3623     # If the command succeeds qemu prints: "OK"
3624     return 1 if $ret =~ m/OK/s;
3625
3626     die "adding drive failed: $ret\n";
3627 }
3628
3629 sub qemu_drivedel {
3630     my($vmid, $deviceid) = @_;
3631
3632     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3633     $ret =~ s/^\s+//;
3634
3635     return 1 if $ret eq "";
3636
3637     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3638     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3639
3640     die "deleting drive $deviceid failed : $ret\n";
3641 }
3642
3643 sub qemu_deviceaddverify {
3644     my ($vmid, $deviceid) = @_;
3645
3646     for (my $i = 0; $i <= 5; $i++) {
3647          my $devices_list = vm_devices_list($vmid);
3648          return 1 if defined($devices_list->{$deviceid});
3649          sleep 1;
3650     }
3651
3652     die "error on hotplug device '$deviceid'\n";
3653 }
3654
3655
3656 sub qemu_devicedelverify {
3657     my ($vmid, $deviceid) = @_;
3658
3659     # need to verify that the device is correctly removed as device_del
3660     # is async and empty return is not reliable
3661
3662     for (my $i = 0; $i <= 5; $i++) {
3663          my $devices_list = vm_devices_list($vmid);
3664          return 1 if !defined($devices_list->{$deviceid});
3665          sleep 1;
3666     }
3667
3668     die "error on hot-unplugging device '$deviceid'\n";
3669 }
3670
3671 sub qemu_findorcreatescsihw {
3672     my ($storecfg, $conf, $vmid, $device) = @_;
3673
3674     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3675
3676     my $scsihwid="$controller_prefix$controller";
3677     my $devices_list = vm_devices_list($vmid);
3678
3679     if(!defined($devices_list->{$scsihwid})) {
3680         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3681     }
3682
3683     return 1;
3684 }
3685
3686 sub qemu_deletescsihw {
3687     my ($conf, $vmid, $opt) = @_;
3688
3689     my $device = parse_drive($opt, $conf->{$opt});
3690
3691     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3692         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3693         return 1;
3694     }
3695
3696     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3697
3698     my $devices_list = vm_devices_list($vmid);
3699     foreach my $opt (keys %{$devices_list}) {
3700         if (PVE::QemuServer::is_valid_drivename($opt)) {
3701             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3702             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3703                 return 1;
3704             }
3705         }
3706     }
3707
3708     my $scsihwid="scsihw$controller";
3709
3710     vm_deviceunplug($vmid, $conf, $scsihwid);
3711
3712     return 1;
3713 }
3714
3715 sub qemu_add_pci_bridge {
3716     my ($storecfg, $conf, $vmid, $device) = @_;
3717
3718     my $bridges = {};
3719
3720     my $bridgeid;
3721
3722     print_pci_addr($device, $bridges);
3723
3724     while (my ($k, $v) = each %$bridges) {
3725         $bridgeid = $k;
3726     }
3727     return 1 if !defined($bridgeid) || $bridgeid < 1;
3728
3729     my $bridge = "pci.$bridgeid";
3730     my $devices_list = vm_devices_list($vmid);
3731
3732     if (!defined($devices_list->{$bridge})) {
3733         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3734     }
3735
3736     return 1;
3737 }
3738
3739 sub qemu_set_link_status {
3740     my ($vmid, $device, $up) = @_;
3741
3742     vm_mon_cmd($vmid, "set_link", name => $device,
3743                up => $up ? JSON::true : JSON::false);
3744 }
3745
3746 sub qemu_netdevadd {
3747     my ($vmid, $conf, $device, $deviceid) = @_;
3748
3749     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3750     my %options =  split(/[=,]/, $netdev);
3751
3752     vm_mon_cmd($vmid, "netdev_add",  %options);
3753     return 1;
3754 }
3755
3756 sub qemu_netdevdel {
3757     my ($vmid, $deviceid) = @_;
3758
3759     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3760 }
3761
3762 sub qemu_cpu_hotplug {
3763     my ($vmid, $conf, $vcpus) = @_;
3764
3765     my $sockets = 1;
3766     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3767     $sockets = $conf->{sockets} if  $conf->{sockets};
3768     my $cores = $conf->{cores} || 1;
3769     my $maxcpus = $sockets * $cores;
3770
3771     $vcpus = $maxcpus if !$vcpus;
3772
3773     die "you can't add more vcpus than maxcpus\n"
3774         if $vcpus > $maxcpus;
3775
3776     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3777     die "online cpu unplug is not yet possible\n"
3778         if $vcpus < $currentvcpus;
3779
3780     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3781     die "vcpus in running vm is different than configuration\n"
3782         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3783
3784     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3785         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3786     }
3787 }
3788
3789 sub qemu_memory_hotplug {
3790     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3791
3792     return $value if !check_running($vmid);
3793
3794     my $memory = $conf->{memory} || $defaults->{memory};
3795     $value = $defaults->{memory} if !$value;
3796     return $value if $value == $memory;
3797
3798     my $static_memory = $STATICMEM;
3799     my $dimm_memory = $memory - $static_memory;
3800
3801     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3802     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3803
3804
3805     my $sockets = 1;
3806     $sockets = $conf->{sockets} if $conf->{sockets};
3807
3808     if($value > $memory) {
3809
3810         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3811             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3812
3813                 return if $current_size <= $conf->{memory};
3814
3815                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3816                 if (my $err = $@) {
3817                     eval { qemu_objectdel($vmid, "mem-$name"); };
3818                     die $err;
3819                 }
3820
3821                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3822                 if (my $err = $@) {
3823                     eval { qemu_objectdel($vmid, "mem-$name"); };
3824                     die $err;
3825                 }
3826                 #update conf after each succesful module hotplug
3827                 $conf->{memory} = $current_size;
3828                 PVE::QemuConfig->write_config($vmid, $conf);
3829         });
3830
3831     } else {
3832
3833         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3834             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3835
3836                 return if $current_size >= $conf->{memory};
3837                 print "try to unplug memory dimm $name\n";
3838
3839                 my $retry = 0;
3840                 while (1) {
3841                     eval { qemu_devicedel($vmid, $name) };
3842                     sleep 3;
3843                     my $dimm_list = qemu_dimm_list($vmid);
3844                     last if !$dimm_list->{$name};
3845                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3846                     $retry++;
3847                 }
3848
3849                 #update conf after each succesful module unplug
3850                 $conf->{memory} = $current_size;
3851
3852                 eval { qemu_objectdel($vmid, "mem-$name"); };
3853                 PVE::QemuConfig->write_config($vmid, $conf);
3854         });
3855     }
3856 }
3857
3858 sub qemu_dimm_list {
3859     my ($vmid) = @_;
3860
3861     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3862     my $dimms = {};
3863
3864     foreach my $dimm (@$dimmarray) {
3865
3866         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3867         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3868         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3869         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3870         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3871     }
3872     return $dimms;
3873 }
3874
3875 sub qemu_block_set_io_throttle {
3876     my ($vmid, $deviceid,
3877         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3878         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3879
3880     return if !check_running($vmid) ;
3881
3882     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3883         bps => int($bps),
3884         bps_rd => int($bps_rd),
3885         bps_wr => int($bps_wr),
3886         iops => int($iops),
3887         iops_rd => int($iops_rd),
3888         iops_wr => int($iops_wr),
3889         bps_max => int($bps_max),
3890         bps_rd_max => int($bps_rd_max),
3891         bps_wr_max => int($bps_wr_max),
3892         iops_max => int($iops_max),
3893         iops_rd_max => int($iops_rd_max),
3894         iops_wr_max => int($iops_wr_max)
3895     );
3896
3897 }
3898
3899 # old code, only used to shutdown old VM after update
3900 sub __read_avail {
3901     my ($fh, $timeout) = @_;
3902
3903     my $sel = new IO::Select;
3904     $sel->add($fh);
3905
3906     my $res = '';
3907     my $buf;
3908
3909     my @ready;
3910     while (scalar (@ready = $sel->can_read($timeout))) {
3911         my $count;
3912         if ($count = $fh->sysread($buf, 8192)) {
3913             if ($buf =~ /^(.*)\(qemu\) $/s) {
3914                 $res .= $1;
3915                 last;
3916             } else {
3917                 $res .= $buf;
3918             }
3919         } else {
3920             if (!defined($count)) {
3921                 die "$!\n";
3922             }
3923             last;
3924         }
3925     }
3926
3927     die "monitor read timeout\n" if !scalar(@ready);
3928
3929     return $res;
3930 }
3931
3932 # old code, only used to shutdown old VM after update
3933 sub vm_monitor_command {
3934     my ($vmid, $cmdstr, $nocheck) = @_;
3935
3936     my $res;
3937
3938     eval {
3939         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3940
3941         my $sname = "${var_run_tmpdir}/$vmid.mon";
3942
3943         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3944             die "unable to connect to VM $vmid socket - $!\n";
3945
3946         my $timeout = 3;
3947
3948         # hack: migrate sometime blocks the monitor (when migrate_downtime
3949         # is set)
3950         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3951             $timeout = 60*60; # 1 hour
3952         }
3953
3954         # read banner;
3955         my $data = __read_avail($sock, $timeout);
3956
3957         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3958             die "got unexpected qemu monitor banner\n";
3959         }
3960
3961         my $sel = new IO::Select;
3962         $sel->add($sock);
3963
3964         if (!scalar(my @ready = $sel->can_write($timeout))) {
3965             die "monitor write error - timeout";
3966         }
3967
3968         my $fullcmd = "$cmdstr\r";
3969
3970         # syslog('info', "VM $vmid monitor command: $cmdstr");
3971
3972         my $b;
3973         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3974             die "monitor write error - $!";
3975         }
3976
3977         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3978
3979         $timeout = 20;
3980
3981         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3982             $timeout = 60*60; # 1 hour
3983         } elsif ($cmdstr =~ m/^(eject|change)/) {
3984             $timeout = 60; # note: cdrom mount command is slow
3985         }
3986         if ($res = __read_avail($sock, $timeout)) {
3987
3988             my @lines = split("\r?\n", $res);
3989
3990             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3991
3992             $res = join("\n", @lines);
3993             $res .= "\n";
3994         }
3995     };
3996
3997     my $err = $@;
3998
3999     if ($err) {
4000         syslog("err", "VM $vmid monitor command failed - $err");
4001         die $err;
4002     }
4003
4004     return $res;
4005 }
4006
4007 sub qemu_block_resize {
4008     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4009
4010     my $running = check_running($vmid);
4011
4012     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4013
4014     return if !$running;
4015
4016     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4017
4018 }
4019
4020 sub qemu_volume_snapshot {
4021     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4022
4023     my $running = check_running($vmid);
4024
4025     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4026         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4027     } else {
4028         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4029     }
4030 }
4031
4032 sub qemu_volume_snapshot_delete {
4033     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4034
4035     my $running = check_running($vmid);
4036
4037     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4038
4039     return if !$running;
4040
4041     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4042 }
4043
4044 sub set_migration_caps {
4045     my ($vmid) = @_;
4046
4047     my $cap_ref = [];
4048
4049     my $enabled_cap = {
4050         "auto-converge" => 1,
4051         "xbzrle" => 1,
4052         "x-rdma-pin-all" => 0,
4053         "zero-blocks" => 0,
4054         "compress" => 0
4055     };
4056
4057     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4058
4059     for my $supported_capability (@$supported_capabilities) {
4060         push @$cap_ref, {
4061             capability => $supported_capability->{capability},
4062             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4063         };
4064     }
4065
4066     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4067 }
4068
4069 my $fast_plug_option = {
4070     'lock' => 1,
4071     'name' => 1,
4072     'onboot' => 1,
4073     'shares' => 1,
4074     'startup' => 1,
4075     'description' => 1,
4076 };
4077
4078 # hotplug changes in [PENDING]
4079 # $selection hash can be used to only apply specified options, for
4080 # example: { cores => 1 } (only apply changed 'cores')
4081 # $errors ref is used to return error messages
4082 sub vmconfig_hotplug_pending {
4083     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4084
4085     my $defaults = load_defaults();
4086
4087     # commit values which do not have any impact on running VM first
4088     # Note: those option cannot raise errors, we we do not care about
4089     # $selection and always apply them.
4090
4091     my $add_error = sub {
4092         my ($opt, $msg) = @_;
4093         $errors->{$opt} = "hotplug problem - $msg";
4094     };
4095
4096     my $changes = 0;
4097     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4098         if ($fast_plug_option->{$opt}) {
4099             $conf->{$opt} = $conf->{pending}->{$opt};
4100             delete $conf->{pending}->{$opt};
4101             $changes = 1;
4102         }
4103     }
4104
4105     if ($changes) {
4106         PVE::QemuConfig->write_config($vmid, $conf);
4107         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4108     }
4109
4110     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4111
4112     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4113     while (my ($opt, $force) = each %$pending_delete_hash) {
4114         next if $selection && !$selection->{$opt};
4115         eval {
4116             if ($opt eq 'hotplug') {
4117                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4118             } elsif ($opt eq 'tablet') {
4119                 die "skip\n" if !$hotplug_features->{usb};
4120                 if ($defaults->{tablet}) {
4121                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4122                 } else {
4123                     vm_deviceunplug($vmid, $conf, $opt);
4124                 }
4125             } elsif ($opt eq 'vcpus') {
4126                 die "skip\n" if !$hotplug_features->{cpu};
4127                 qemu_cpu_hotplug($vmid, $conf, undef);
4128             } elsif ($opt eq 'balloon') {
4129                 # enable balloon device is not hotpluggable
4130                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4131             } elsif ($fast_plug_option->{$opt}) {
4132                 # do nothing
4133             } elsif ($opt =~ m/^net(\d+)$/) {
4134                 die "skip\n" if !$hotplug_features->{network};
4135                 vm_deviceunplug($vmid, $conf, $opt);
4136             } elsif (is_valid_drivename($opt)) {
4137                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4138                 vm_deviceunplug($vmid, $conf, $opt);
4139                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4140             } elsif ($opt =~ m/^memory$/) {
4141                 die "skip\n" if !$hotplug_features->{memory};
4142                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4143             } elsif ($opt eq 'cpuunits') {
4144                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4145             } elsif ($opt eq 'cpulimit') {
4146                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4147             } else {
4148                 die "skip\n";
4149             }
4150         };
4151         if (my $err = $@) {
4152             &$add_error($opt, $err) if $err ne "skip\n";
4153         } else {
4154             # save new config if hotplug was successful
4155             delete $conf->{$opt};
4156             vmconfig_undelete_pending_option($conf, $opt);
4157             PVE::QemuConfig->write_config($vmid, $conf);
4158             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4159         }
4160     }
4161
4162     foreach my $opt (keys %{$conf->{pending}}) {
4163         next if $selection && !$selection->{$opt};
4164         my $value = $conf->{pending}->{$opt};
4165         eval {
4166             if ($opt eq 'hotplug') {
4167                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4168             } elsif ($opt eq 'tablet') {
4169                 die "skip\n" if !$hotplug_features->{usb};
4170                 if ($value == 1) {
4171                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4172                 } elsif ($value == 0) {
4173                     vm_deviceunplug($vmid, $conf, $opt);
4174                 }
4175             } elsif ($opt eq 'vcpus') {
4176                 die "skip\n" if !$hotplug_features->{cpu};
4177                 qemu_cpu_hotplug($vmid, $conf, $value);
4178             } elsif ($opt eq 'balloon') {
4179                 # enable/disable balloning device is not hotpluggable
4180                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4181                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4182                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4183
4184                 # allow manual ballooning if shares is set to zero
4185                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4186                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4187                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4188                 }
4189             } elsif ($opt =~ m/^net(\d+)$/) {
4190                 # some changes can be done without hotplug
4191                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4192                                     $vmid, $opt, $value);
4193             } elsif (is_valid_drivename($opt)) {
4194                 # some changes can be done without hotplug
4195                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4196                                      $vmid, $opt, $value, 1);
4197             } elsif ($opt =~ m/^memory$/) { #dimms
4198                 die "skip\n" if !$hotplug_features->{memory};
4199                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4200             } elsif ($opt eq 'cpuunits') {
4201                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4202             } elsif ($opt eq 'cpulimit') {
4203                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4204                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4205             } else {
4206                 die "skip\n";  # skip non-hot-pluggable options
4207             }
4208         };
4209         if (my $err = $@) {
4210             &$add_error($opt, $err) if $err ne "skip\n";
4211         } else {
4212             # save new config if hotplug was successful
4213             $conf->{$opt} = $value;
4214             delete $conf->{pending}->{$opt};
4215             PVE::QemuConfig->write_config($vmid, $conf);
4216             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4217         }
4218     }
4219 }
4220
4221 sub try_deallocate_drive {
4222     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4223
4224     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4225         my $volid = $drive->{file};
4226         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4227             my $sid = PVE::Storage::parse_volume_id($volid);
4228             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4229
4230             # check if the disk is really unused
4231             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4232                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4233             PVE::Storage::vdisk_free($storecfg, $volid);
4234             return 1;
4235         } else {
4236             # If vm is not owner of this disk remove from config
4237             return 1;
4238         }
4239     }
4240
4241     return undef;
4242 }
4243
4244 sub vmconfig_delete_or_detach_drive {
4245     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4246
4247     my $drive = parse_drive($opt, $conf->{$opt});
4248
4249     my $rpcenv = PVE::RPCEnvironment::get();
4250     my $authuser = $rpcenv->get_user();
4251
4252     if ($force) {
4253         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4254         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4255     } else {
4256         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4257     }
4258 }
4259
4260 sub vmconfig_apply_pending {
4261     my ($vmid, $conf, $storecfg) = @_;
4262
4263     # cold plug
4264
4265     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4266     while (my ($opt, $force) = each %$pending_delete_hash) {
4267         die "internal error" if $opt =~ m/^unused/;
4268         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4269         if (!defined($conf->{$opt})) {
4270             vmconfig_undelete_pending_option($conf, $opt);
4271             PVE::QemuConfig->write_config($vmid, $conf);
4272         } elsif (is_valid_drivename($opt)) {
4273             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4274             vmconfig_undelete_pending_option($conf, $opt);
4275             delete $conf->{$opt};
4276             PVE::QemuConfig->write_config($vmid, $conf);
4277         } else {
4278             vmconfig_undelete_pending_option($conf, $opt);
4279             delete $conf->{$opt};
4280             PVE::QemuConfig->write_config($vmid, $conf);
4281         }
4282     }
4283
4284     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4285
4286     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4287         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4288
4289         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4290             # skip if nothing changed
4291         } elsif (is_valid_drivename($opt)) {
4292             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4293                 if defined($conf->{$opt});
4294             $conf->{$opt} = $conf->{pending}->{$opt};
4295         } else {
4296             $conf->{$opt} = $conf->{pending}->{$opt};
4297         }
4298
4299         delete $conf->{pending}->{$opt};
4300         PVE::QemuConfig->write_config($vmid, $conf);
4301     }
4302 }
4303
4304 my $safe_num_ne = sub {
4305     my ($a, $b) = @_;
4306
4307     return 0 if !defined($a) && !defined($b);
4308     return 1 if !defined($a);
4309     return 1 if !defined($b);
4310
4311     return $a != $b;
4312 };
4313
4314 my $safe_string_ne = sub {
4315     my ($a, $b) = @_;
4316
4317     return 0 if !defined($a) && !defined($b);
4318     return 1 if !defined($a);
4319     return 1 if !defined($b);
4320
4321     return $a ne $b;
4322 };
4323
4324 sub vmconfig_update_net {
4325     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4326
4327     my $newnet = parse_net($value);
4328
4329     if ($conf->{$opt}) {
4330         my $oldnet = parse_net($conf->{$opt});
4331
4332         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4333             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4334             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4335             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4336
4337             # for non online change, we try to hot-unplug
4338             die "skip\n" if !$hotplug;
4339             vm_deviceunplug($vmid, $conf, $opt);
4340         } else {
4341
4342             die "internal error" if $opt !~ m/net(\d+)/;
4343             my $iface = "tap${vmid}i$1";
4344
4345             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4346                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4347                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4348                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4349                 PVE::Network::tap_unplug($iface);
4350                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4351             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4352                 # Rate can be applied on its own but any change above needs to
4353                 # include the rate in tap_plug since OVS resets everything.
4354                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4355             }
4356
4357             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4358                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4359             }
4360
4361             return 1;
4362         }
4363     }
4364
4365     if ($hotplug) {
4366         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4367     } else {
4368         die "skip\n";
4369     }
4370 }
4371
4372 sub vmconfig_update_disk {
4373     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4374
4375     # fixme: do we need force?
4376
4377     my $drive = parse_drive($opt, $value);
4378
4379     if ($conf->{$opt}) {
4380
4381         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4382
4383             my $media = $drive->{media} || 'disk';
4384             my $oldmedia = $old_drive->{media} || 'disk';
4385             die "unable to change media type\n" if $media ne $oldmedia;
4386
4387             if (!drive_is_cdrom($old_drive)) {
4388
4389                 if ($drive->{file} ne $old_drive->{file}) {
4390
4391                     die "skip\n" if !$hotplug;
4392
4393                     # unplug and register as unused
4394                     vm_deviceunplug($vmid, $conf, $opt);
4395                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4396
4397                 } else {
4398                     # update existing disk
4399
4400                     # skip non hotpluggable value
4401                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4402                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4403                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4404                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4405                         die "skip\n";
4406                     }
4407
4408                     # apply throttle
4409                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4410                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4411                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4412                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4413                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4414                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4415                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4416                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4417                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4418                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4419                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4420                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4421
4422                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4423                                                    ($drive->{mbps} || 0)*1024*1024,
4424                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4425                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4426                                                    $drive->{iops} || 0,
4427                                                    $drive->{iops_rd} || 0,
4428                                                    $drive->{iops_wr} || 0,
4429                                                    ($drive->{mbps_max} || 0)*1024*1024,
4430                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4431                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4432                                                    $drive->{iops_max} || 0,
4433                                                    $drive->{iops_rd_max} || 0,
4434                                                    $drive->{iops_wr_max} || 0);
4435
4436                     }
4437
4438                     return 1;
4439                 }
4440
4441             } else { # cdrom
4442
4443                 if ($drive->{file} eq 'none') {
4444                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4445                 } else {
4446                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4447                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4448                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4449                 }
4450
4451                 return 1;
4452             }
4453         }
4454     }
4455
4456     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4457     # hotplug new disks
4458     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4459     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4460 }
4461
4462 sub vm_start {
4463     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4464         $forcemachine, $spice_ticket) = @_;
4465
4466     PVE::QemuConfig->lock_config($vmid, sub {
4467         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4468
4469         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4470
4471         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4472
4473         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4474
4475         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4476             vmconfig_apply_pending($vmid, $conf, $storecfg);
4477             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4478         }
4479
4480         my $defaults = load_defaults();
4481
4482         # set environment variable useful inside network script
4483         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4484
4485         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4486
4487         my $migrate_port = 0;
4488         my $migrate_uri;
4489         if ($statefile) {
4490             if ($statefile eq 'tcp') {
4491                 my $localip = "localhost";
4492                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4493                 my $nodename = PVE::INotify::nodename();
4494                 if ($datacenterconf->{migration_unsecure}) {
4495                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4496                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4497                 }
4498                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4499                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4500                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4501                 push @$cmd, '-incoming', $migrate_uri;
4502                 push @$cmd, '-S';
4503             } else {
4504                 push @$cmd, '-loadstate', $statefile;
4505             }
4506         } elsif ($paused) {
4507             push @$cmd, '-S';
4508         }
4509
4510         # host pci devices
4511         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4512           my $d = parse_hostpci($conf->{"hostpci$i"});
4513           next if !$d;
4514           my $pcidevices = $d->{pciid};
4515           foreach my $pcidevice (@$pcidevices) {
4516                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4517
4518                 my $info = pci_device_info("0000:$pciid");
4519                 die "IOMMU not present\n" if !check_iommu_support();
4520                 die "no pci device info for device '$pciid'\n" if !$info;
4521                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4522                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4523           }
4524         }
4525
4526         PVE::Storage::activate_volumes($storecfg, $vollist);
4527
4528         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4529                     umask => 0077); };
4530
4531         if (my $err = $@) {
4532             # deactivate volumes if start fails
4533             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4534             die "start failed: $err";
4535         }
4536
4537         print "migration listens on $migrate_uri\n" if $migrate_uri;
4538
4539         if ($statefile && $statefile ne 'tcp')  {
4540             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4541             warn $@ if $@;
4542         }
4543
4544         if ($migratedfrom) {
4545
4546             eval {
4547                 set_migration_caps($vmid);
4548             };
4549             warn $@ if $@;
4550
4551             if ($spice_port) {
4552                 print "spice listens on port $spice_port\n";
4553                 if ($spice_ticket) {
4554                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4555                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4556                 }
4557             }
4558
4559         } else {
4560
4561             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4562                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4563                     if $conf->{balloon};
4564             }
4565
4566             foreach my $opt (keys %$conf) {
4567                 next if $opt !~  m/^net\d+$/;
4568                 my $nicconf = parse_net($conf->{$opt});
4569                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4570             }
4571         }
4572
4573         vm_mon_cmd_nocheck($vmid, 'qom-set',
4574                     path => "machine/peripheral/balloon0",
4575                     property => "guest-stats-polling-interval",
4576                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4577
4578     });
4579 }
4580
4581 sub vm_mon_cmd {
4582     my ($vmid, $execute, %params) = @_;
4583
4584     my $cmd = { execute => $execute, arguments => \%params };
4585     vm_qmp_command($vmid, $cmd);
4586 }
4587
4588 sub vm_mon_cmd_nocheck {
4589     my ($vmid, $execute, %params) = @_;
4590
4591     my $cmd = { execute => $execute, arguments => \%params };
4592     vm_qmp_command($vmid, $cmd, 1);
4593 }
4594
4595 sub vm_qmp_command {
4596     my ($vmid, $cmd, $nocheck) = @_;
4597
4598     my $res;
4599
4600     my $timeout;
4601     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4602         $timeout = $cmd->{arguments}->{timeout};
4603         delete $cmd->{arguments}->{timeout};
4604     }
4605
4606     eval {
4607         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4608         my $sname = qmp_socket($vmid);
4609         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4610             my $qmpclient = PVE::QMPClient->new();
4611
4612             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4613         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4614             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4615                 if scalar(%{$cmd->{arguments}});
4616             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4617         } else {
4618             die "unable to open monitor socket\n";
4619         }
4620     };
4621     if (my $err = $@) {
4622         syslog("err", "VM $vmid qmp command failed - $err");
4623         die $err;
4624     }
4625
4626     return $res;
4627 }
4628
4629 sub vm_human_monitor_command {
4630     my ($vmid, $cmdline) = @_;
4631
4632     my $res;
4633
4634     my $cmd = {
4635         execute => 'human-monitor-command',
4636         arguments => { 'command-line' => $cmdline},
4637     };
4638
4639     return vm_qmp_command($vmid, $cmd);
4640 }
4641
4642 sub vm_commandline {
4643     my ($storecfg, $vmid) = @_;
4644
4645     my $conf = PVE::QemuConfig->load_config($vmid);
4646
4647     my $defaults = load_defaults();
4648
4649     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4650
4651     return join(' ', @$cmd);
4652 }
4653
4654 sub vm_reset {
4655     my ($vmid, $skiplock) = @_;
4656
4657     PVE::QemuConfig->lock_config($vmid, sub {
4658
4659         my $conf = PVE::QemuConfig->load_config($vmid);
4660
4661         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4662
4663         vm_mon_cmd($vmid, "system_reset");
4664     });
4665 }
4666
4667 sub get_vm_volumes {
4668     my ($conf) = @_;
4669
4670     my $vollist = [];
4671     foreach_volid($conf, sub {
4672         my ($volid, $is_cdrom) = @_;
4673
4674         return if $volid =~ m|^/|;
4675
4676         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4677         return if !$sid;
4678
4679         push @$vollist, $volid;
4680     });
4681
4682     return $vollist;
4683 }
4684
4685 sub vm_stop_cleanup {
4686     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4687
4688     eval {
4689
4690         if (!$keepActive) {
4691             my $vollist = get_vm_volumes($conf);
4692             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4693         }
4694
4695         foreach my $ext (qw(mon qmp pid vnc qga)) {
4696             unlink "/var/run/qemu-server/${vmid}.$ext";
4697         }
4698
4699         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4700     };
4701     warn $@ if $@; # avoid errors - just warn
4702 }
4703
4704 # Note: use $nockeck to skip tests if VM configuration file exists.
4705 # We need that when migration VMs to other nodes (files already moved)
4706 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4707 sub vm_stop {
4708     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4709
4710     $force = 1 if !defined($force) && !$shutdown;
4711
4712     if ($migratedfrom){
4713         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4714         kill 15, $pid if $pid;
4715         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4716         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4717         return;
4718     }
4719
4720     PVE::QemuConfig->lock_config($vmid, sub {
4721
4722         my $pid = check_running($vmid, $nocheck);
4723         return if !$pid;
4724
4725         my $conf;
4726         if (!$nocheck) {
4727             $conf = PVE::QemuConfig->load_config($vmid);
4728             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4729             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4730                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4731                 $timeout = $opts->{down} if $opts->{down};
4732             }
4733         }
4734
4735         $timeout = 60 if !defined($timeout);
4736
4737         eval {
4738             if ($shutdown) {
4739                 if (defined($conf) && $conf->{agent}) {
4740                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4741                 } else {
4742                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4743                 }
4744             } else {
4745                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4746             }
4747         };
4748         my $err = $@;
4749
4750         if (!$err) {
4751             my $count = 0;
4752             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4753                 $count++;
4754                 sleep 1;
4755             }
4756
4757             if ($count >= $timeout) {
4758                 if ($force) {
4759                     warn "VM still running - terminating now with SIGTERM\n";
4760                     kill 15, $pid;
4761                 } else {
4762                     die "VM quit/powerdown failed - got timeout\n";
4763                 }
4764             } else {
4765                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4766                 return;
4767             }
4768         } else {
4769             if ($force) {
4770                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4771                 kill 15, $pid;
4772             } else {
4773                 die "VM quit/powerdown failed\n";
4774             }
4775         }
4776
4777         # wait again
4778         $timeout = 10;
4779
4780         my $count = 0;
4781         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4782             $count++;
4783             sleep 1;
4784         }
4785
4786         if ($count >= $timeout) {
4787             warn "VM still running - terminating now with SIGKILL\n";
4788             kill 9, $pid;
4789             sleep 1;
4790         }
4791
4792         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4793    });
4794 }
4795
4796 sub vm_suspend {
4797     my ($vmid, $skiplock) = @_;
4798
4799     PVE::QemuConfig->lock_config($vmid, sub {
4800
4801         my $conf = PVE::QemuConfig->load_config($vmid);
4802
4803         PVE::QemuConfig->check_lock($conf)
4804             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4805
4806         vm_mon_cmd($vmid, "stop");
4807     });
4808 }
4809
4810 sub vm_resume {
4811     my ($vmid, $skiplock, $nocheck) = @_;
4812
4813     PVE::QemuConfig->lock_config($vmid, sub {
4814
4815         if (!$nocheck) {
4816
4817             my $conf = PVE::QemuConfig->load_config($vmid);
4818
4819             PVE::QemuConfig->check_lock($conf)
4820                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4821
4822             vm_mon_cmd($vmid, "cont");
4823
4824         } else {
4825             vm_mon_cmd_nocheck($vmid, "cont");
4826         }
4827     });
4828 }
4829
4830 sub vm_sendkey {
4831     my ($vmid, $skiplock, $key) = @_;
4832
4833     PVE::QemuConfig->lock_config($vmid, sub {
4834
4835         my $conf = PVE::QemuConfig->load_config($vmid);
4836
4837         # there is no qmp command, so we use the human monitor command
4838         vm_human_monitor_command($vmid, "sendkey $key");
4839     });
4840 }
4841
4842 sub vm_destroy {
4843     my ($storecfg, $vmid, $skiplock) = @_;
4844
4845     PVE::QemuConfig->lock_config($vmid, sub {
4846
4847         my $conf = PVE::QemuConfig->load_config($vmid);
4848
4849         if (!check_running($vmid)) {
4850             destroy_vm($storecfg, $vmid, undef, $skiplock);
4851         } else {
4852             die "VM $vmid is running - destroy failed\n";
4853         }
4854     });
4855 }
4856
4857 # pci helpers
4858
4859 sub file_write {
4860     my ($filename, $buf) = @_;
4861
4862     my $fh = IO::File->new($filename, "w");
4863     return undef if !$fh;
4864
4865     my $res = print $fh $buf;
4866
4867     $fh->close();
4868
4869     return $res;
4870 }
4871
4872 sub pci_device_info {
4873     my ($name) = @_;
4874
4875     my $res;
4876
4877     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4878     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4879
4880     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4881     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4882
4883     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4884     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4885
4886     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4887     return undef if !defined($product) || $product !~ s/^0x//;
4888
4889     $res = {
4890         name => $name,
4891         vendor => $vendor,
4892         product => $product,
4893         domain => $domain,
4894         bus => $bus,
4895         slot => $slot,
4896         func => $func,
4897         irq => $irq,
4898         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4899     };
4900
4901     return $res;
4902 }
4903
4904 sub pci_dev_reset {
4905     my ($dev) = @_;
4906
4907     my $name = $dev->{name};
4908
4909     my $fn = "$pcisysfs/devices/$name/reset";
4910
4911     return file_write($fn, "1");
4912 }
4913
4914 sub pci_dev_bind_to_vfio {
4915     my ($dev) = @_;
4916
4917     my $name = $dev->{name};
4918
4919     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4920
4921     if (!-d $vfio_basedir) {
4922         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4923     }
4924     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4925
4926     my $testdir = "$vfio_basedir/$name";
4927     return 1 if -d $testdir;
4928
4929     my $data = "$dev->{vendor} $dev->{product}";
4930     return undef if !file_write("$vfio_basedir/new_id", $data);
4931
4932     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4933     if (!file_write($fn, $name)) {
4934         return undef if -f $fn;
4935     }
4936
4937     $fn = "$vfio_basedir/bind";
4938     if (! -d $testdir) {
4939         return undef if !file_write($fn, $name);
4940     }
4941
4942     return -d $testdir;
4943 }
4944
4945 sub pci_dev_group_bind_to_vfio {
4946     my ($pciid) = @_;
4947
4948     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4949
4950     if (!-d $vfio_basedir) {
4951         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4952     }
4953     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4954
4955     # get IOMMU group devices
4956     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4957       my @devs = grep /^0000:/, readdir($D);
4958     closedir($D);
4959
4960     foreach my $pciid (@devs) {
4961         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4962
4963         # pci bridges, switches or root ports are not supported
4964         # they have a pci_bus subdirectory so skip them
4965         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4966
4967         my $info = pci_device_info($1);
4968         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4969     }
4970
4971     return 1;
4972 }
4973
4974 sub print_pci_addr {
4975     my ($id, $bridges) = @_;
4976
4977     my $res = '';
4978     my $devices = {
4979         piix3 => { bus => 0, addr => 1 },
4980         #addr2 : first videocard
4981         balloon0 => { bus => 0, addr => 3 },
4982         watchdog => { bus => 0, addr => 4 },
4983         scsihw0 => { bus => 0, addr => 5 },
4984         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4985         scsihw1 => { bus => 0, addr => 6 },
4986         ahci0 => { bus => 0, addr => 7 },
4987         qga0 => { bus => 0, addr => 8 },
4988         spice => { bus => 0, addr => 9 },
4989         virtio0 => { bus => 0, addr => 10 },
4990         virtio1 => { bus => 0, addr => 11 },
4991         virtio2 => { bus => 0, addr => 12 },
4992         virtio3 => { bus => 0, addr => 13 },
4993         virtio4 => { bus => 0, addr => 14 },
4994         virtio5 => { bus => 0, addr => 15 },
4995         hostpci0 => { bus => 0, addr => 16 },
4996         hostpci1 => { bus => 0, addr => 17 },
4997         net0 => { bus => 0, addr => 18 },
4998         net1 => { bus => 0, addr => 19 },
4999         net2 => { bus => 0, addr => 20 },
5000         net3 => { bus => 0, addr => 21 },
5001         net4 => { bus => 0, addr => 22 },
5002         net5 => { bus => 0, addr => 23 },
5003         vga1 => { bus => 0, addr => 24 },
5004         vga2 => { bus => 0, addr => 25 },
5005         vga3 => { bus => 0, addr => 26 },
5006         hostpci2 => { bus => 0, addr => 27 },
5007         hostpci3 => { bus => 0, addr => 28 },
5008         #addr29 : usb-host (pve-usb.cfg)
5009         'pci.1' => { bus => 0, addr => 30 },
5010         'pci.2' => { bus => 0, addr => 31 },
5011         'net6' => { bus => 1, addr => 1 },
5012         'net7' => { bus => 1, addr => 2 },
5013         'net8' => { bus => 1, addr => 3 },
5014         'net9' => { bus => 1, addr => 4 },
5015         'net10' => { bus => 1, addr => 5 },
5016         'net11' => { bus => 1, addr => 6 },
5017         'net12' => { bus => 1, addr => 7 },
5018         'net13' => { bus => 1, addr => 8 },
5019         'net14' => { bus => 1, addr => 9 },
5020         'net15' => { bus => 1, addr => 10 },
5021         'net16' => { bus => 1, addr => 11 },
5022         'net17' => { bus => 1, addr => 12 },
5023         'net18' => { bus => 1, addr => 13 },
5024         'net19' => { bus => 1, addr => 14 },
5025         'net20' => { bus => 1, addr => 15 },
5026         'net21' => { bus => 1, addr => 16 },
5027         'net22' => { bus => 1, addr => 17 },
5028         'net23' => { bus => 1, addr => 18 },
5029         'net24' => { bus => 1, addr => 19 },
5030         'net25' => { bus => 1, addr => 20 },
5031         'net26' => { bus => 1, addr => 21 },
5032         'net27' => { bus => 1, addr => 22 },
5033         'net28' => { bus => 1, addr => 23 },
5034         'net29' => { bus => 1, addr => 24 },
5035         'net30' => { bus => 1, addr => 25 },
5036         'net31' => { bus => 1, addr => 26 },
5037         'xhci' => { bus => 1, addr => 27 },
5038         'virtio6' => { bus => 2, addr => 1 },
5039         'virtio7' => { bus => 2, addr => 2 },
5040         'virtio8' => { bus => 2, addr => 3 },
5041         'virtio9' => { bus => 2, addr => 4 },
5042         'virtio10' => { bus => 2, addr => 5 },
5043         'virtio11' => { bus => 2, addr => 6 },
5044         'virtio12' => { bus => 2, addr => 7 },
5045         'virtio13' => { bus => 2, addr => 8 },
5046         'virtio14' => { bus => 2, addr => 9 },
5047         'virtio15' => { bus => 2, addr => 10 },
5048         'virtioscsi0' => { bus => 3, addr => 1 },
5049         'virtioscsi1' => { bus => 3, addr => 2 },
5050         'virtioscsi2' => { bus => 3, addr => 3 },
5051         'virtioscsi3' => { bus => 3, addr => 4 },
5052         'virtioscsi4' => { bus => 3, addr => 5 },
5053         'virtioscsi5' => { bus => 3, addr => 6 },
5054         'virtioscsi6' => { bus => 3, addr => 7 },
5055         'virtioscsi7' => { bus => 3, addr => 8 },
5056         'virtioscsi8' => { bus => 3, addr => 9 },
5057         'virtioscsi9' => { bus => 3, addr => 10 },
5058         'virtioscsi10' => { bus => 3, addr => 11 },
5059         'virtioscsi11' => { bus => 3, addr => 12 },
5060         'virtioscsi12' => { bus => 3, addr => 13 },
5061         'virtioscsi13' => { bus => 3, addr => 14 },
5062         'virtioscsi14' => { bus => 3, addr => 15 },
5063         'virtioscsi15' => { bus => 3, addr => 16 },
5064         'virtioscsi16' => { bus => 3, addr => 17 },
5065         'virtioscsi17' => { bus => 3, addr => 18 },
5066         'virtioscsi18' => { bus => 3, addr => 19 },
5067         'virtioscsi19' => { bus => 3, addr => 20 },
5068         'virtioscsi20' => { bus => 3, addr => 21 },
5069         'virtioscsi21' => { bus => 3, addr => 22 },
5070         'virtioscsi22' => { bus => 3, addr => 23 },
5071         'virtioscsi23' => { bus => 3, addr => 24 },
5072         'virtioscsi24' => { bus => 3, addr => 25 },
5073         'virtioscsi25' => { bus => 3, addr => 26 },
5074         'virtioscsi26' => { bus => 3, addr => 27 },
5075         'virtioscsi27' => { bus => 3, addr => 28 },
5076         'virtioscsi28' => { bus => 3, addr => 29 },
5077         'virtioscsi29' => { bus => 3, addr => 30 },
5078         'virtioscsi30' => { bus => 3, addr => 31 },
5079
5080     };
5081
5082     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5083            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5084            my $bus = $devices->{$id}->{bus};
5085            $res = ",bus=pci.$bus,addr=$addr";
5086            $bridges->{$bus} = 1 if $bridges;
5087     }
5088     return $res;
5089
5090 }
5091
5092 sub print_pcie_addr {
5093     my ($id) = @_;
5094
5095     my $res = '';
5096     my $devices = {
5097         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5098         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5099         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5100         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5101     };
5102
5103     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5104            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5105            my $bus = $devices->{$id}->{bus};
5106            $res = ",bus=$bus,addr=$addr";
5107     }
5108     return $res;
5109
5110 }
5111
5112 # vzdump restore implementaion
5113
5114 sub tar_archive_read_firstfile {
5115     my $archive = shift;
5116
5117     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5118
5119     # try to detect archive type first
5120     my $pid = open (TMP, "tar tf '$archive'|") ||
5121         die "unable to open file '$archive'\n";
5122     my $firstfile = <TMP>;
5123     kill 15, $pid;
5124     close TMP;
5125
5126     die "ERROR: archive contaions no data\n" if !$firstfile;
5127     chomp $firstfile;
5128
5129     return $firstfile;
5130 }
5131
5132 sub tar_restore_cleanup {
5133     my ($storecfg, $statfile) = @_;
5134
5135     print STDERR "starting cleanup\n";
5136
5137     if (my $fd = IO::File->new($statfile, "r")) {
5138         while (defined(my $line = <$fd>)) {
5139             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5140                 my $volid = $2;
5141                 eval {
5142                     if ($volid =~ m|^/|) {
5143                         unlink $volid || die 'unlink failed\n';
5144                     } else {
5145                         PVE::Storage::vdisk_free($storecfg, $volid);
5146                     }
5147                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5148                 };
5149                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5150             } else {
5151                 print STDERR "unable to parse line in statfile - $line";
5152             }
5153         }
5154         $fd->close();
5155     }
5156 }
5157
5158 sub restore_archive {
5159     my ($archive, $vmid, $user, $opts) = @_;
5160
5161     my $format = $opts->{format};
5162     my $comp;
5163
5164     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5165         $format = 'tar' if !$format;
5166         $comp = 'gzip';
5167     } elsif ($archive =~ m/\.tar$/) {
5168         $format = 'tar' if !$format;
5169     } elsif ($archive =~ m/.tar.lzo$/) {
5170         $format = 'tar' if !$format;
5171         $comp = 'lzop';
5172     } elsif ($archive =~ m/\.vma$/) {
5173         $format = 'vma' if !$format;
5174     } elsif ($archive =~ m/\.vma\.gz$/) {
5175         $format = 'vma' if !$format;
5176         $comp = 'gzip';
5177     } elsif ($archive =~ m/\.vma\.lzo$/) {
5178         $format = 'vma' if !$format;
5179         $comp = 'lzop';
5180     } else {
5181         $format = 'vma' if !$format; # default
5182     }
5183
5184     # try to detect archive format
5185     if ($format eq 'tar') {
5186         return restore_tar_archive($archive, $vmid, $user, $opts);
5187     } else {
5188         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5189     }
5190 }
5191
5192 sub restore_update_config_line {
5193     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5194
5195     return if $line =~ m/^\#qmdump\#/;
5196     return if $line =~ m/^\#vzdump\#/;
5197     return if $line =~ m/^lock:/;
5198     return if $line =~ m/^unused\d+:/;
5199     return if $line =~ m/^parent:/;
5200     return if $line =~ m/^template:/; # restored VM is never a template
5201
5202     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5203         # try to convert old 1.X settings
5204         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5205         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5206             my ($model, $macaddr) = split(/\=/, $devconfig);
5207             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5208             my $net = {
5209                 model => $model,
5210                 bridge => "vmbr$ind",
5211                 macaddr => $macaddr,
5212             };
5213             my $netstr = print_net($net);
5214
5215             print $outfd "net$cookie->{netcount}: $netstr\n";
5216             $cookie->{netcount}++;
5217         }
5218     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5219         my ($id, $netstr) = ($1, $2);
5220         my $net = parse_net($netstr);
5221         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5222         $netstr = print_net($net);
5223         print $outfd "$id: $netstr\n";
5224     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5225         my $virtdev = $1;
5226         my $value = $3;
5227         my $di = parse_drive($virtdev, $value);
5228         if (defined($di->{backup}) && !$di->{backup}) {
5229             print $outfd "#$line";
5230         } elsif ($map->{$virtdev}) {
5231             delete $di->{format}; # format can change on restore
5232             $di->{file} = $map->{$virtdev};
5233             $value = print_drive($vmid, $di);
5234             print $outfd "$virtdev: $value\n";
5235         } else {
5236             print $outfd $line;
5237         }
5238     } else {
5239         print $outfd $line;
5240     }
5241 }
5242
5243 sub scan_volids {
5244     my ($cfg, $vmid) = @_;
5245
5246     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5247
5248     my $volid_hash = {};
5249     foreach my $storeid (keys %$info) {
5250         foreach my $item (@{$info->{$storeid}}) {
5251             next if !($item->{volid} && $item->{size});
5252             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5253             $volid_hash->{$item->{volid}} = $item;
5254         }
5255     }
5256
5257     return $volid_hash;
5258 }
5259
5260 sub is_volume_in_use {
5261     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5262
5263     my $path = PVE::Storage::path($storecfg, $volid);
5264
5265     my $scan_config = sub {
5266         my ($cref, $snapname) = @_;
5267
5268         foreach my $key (keys %$cref) {
5269             my $value = $cref->{$key};
5270             if (is_valid_drivename($key)) {
5271                 next if $skip_drive && $key eq $skip_drive;
5272                 my $drive = parse_drive($key, $value);
5273                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5274                 return 1 if $volid eq $drive->{file};
5275                 if ($drive->{file} =~ m!^/!) {
5276                     return 1 if $drive->{file} eq $path;
5277                 } else {
5278                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5279                     next if !$storeid;
5280                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5281                     next if !$scfg;
5282                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5283                 }
5284             }
5285         }
5286
5287         return 0;
5288     };
5289
5290     return 1 if &$scan_config($conf);
5291
5292     undef $skip_drive;
5293
5294     foreach my $snapname (keys %{$conf->{snapshots}}) {
5295         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5296     }
5297
5298     return 0;
5299 }
5300
5301 sub update_disksize {
5302     my ($vmid, $conf, $volid_hash) = @_;
5303
5304     my $changes;
5305
5306     my $used = {};
5307
5308     # Note: it is allowed to define multiple storages with same path (alias), so
5309     # we need to check both 'volid' and real 'path' (two different volid can point
5310     # to the same path).
5311
5312     my $usedpath = {};
5313
5314     # update size info
5315     foreach my $opt (keys %$conf) {
5316         if (is_valid_drivename($opt)) {
5317             my $drive = parse_drive($opt, $conf->{$opt});
5318             my $volid = $drive->{file};
5319             next if !$volid;
5320
5321             $used->{$volid} = 1;
5322             if ($volid_hash->{$volid} &&
5323                 (my $path = $volid_hash->{$volid}->{path})) {
5324                 $usedpath->{$path} = 1;
5325             }
5326
5327             next if drive_is_cdrom($drive);
5328             next if !$volid_hash->{$volid};
5329
5330             $drive->{size} = $volid_hash->{$volid}->{size};
5331             my $new = print_drive($vmid, $drive);
5332             if ($new ne $conf->{$opt}) {
5333                 $changes = 1;
5334                 $conf->{$opt} = $new;
5335             }
5336         }
5337     }
5338
5339     # remove 'unusedX' entry if volume is used
5340     foreach my $opt (keys %$conf) {
5341         next if $opt !~ m/^unused\d+$/;
5342         my $volid = $conf->{$opt};
5343         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5344         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5345             $changes = 1;
5346             delete $conf->{$opt};
5347         }
5348     }
5349
5350     foreach my $volid (sort keys %$volid_hash) {
5351         next if $volid =~ m/vm-$vmid-state-/;
5352         next if $used->{$volid};
5353         my $path = $volid_hash->{$volid}->{path};
5354         next if !$path; # just to be sure
5355         next if $usedpath->{$path};
5356         $changes = 1;
5357         PVE::QemuConfig->add_unused_volume($conf, $volid);
5358         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5359     }
5360
5361     return $changes;
5362 }
5363
5364 sub rescan {
5365     my ($vmid, $nolock) = @_;
5366
5367     my $cfg = PVE::Storage::config();
5368
5369     my $volid_hash = scan_volids($cfg, $vmid);
5370
5371     my $updatefn =  sub {
5372         my ($vmid) = @_;
5373
5374         my $conf = PVE::QemuConfig->load_config($vmid);
5375
5376         PVE::QemuConfig->check_lock($conf);
5377
5378         my $vm_volids = {};
5379         foreach my $volid (keys %$volid_hash) {
5380             my $info = $volid_hash->{$volid};
5381             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5382         }
5383
5384         my $changes = update_disksize($vmid, $conf, $vm_volids);
5385
5386         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5387     };
5388
5389     if (defined($vmid)) {
5390         if ($nolock) {
5391             &$updatefn($vmid);
5392         } else {
5393             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5394         }
5395     } else {
5396         my $vmlist = config_list();
5397         foreach my $vmid (keys %$vmlist) {
5398             if ($nolock) {
5399                 &$updatefn($vmid);
5400             } else {
5401                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5402             }
5403         }
5404     }
5405 }
5406
5407 sub restore_vma_archive {
5408     my ($archive, $vmid, $user, $opts, $comp) = @_;
5409
5410     my $input = $archive eq '-' ? "<&STDIN" : undef;
5411     my $readfrom = $archive;
5412
5413     my $uncomp = '';
5414     if ($comp) {
5415         $readfrom = '-';
5416         my $qarchive = PVE::Tools::shellquote($archive);
5417         if ($comp eq 'gzip') {
5418             $uncomp = "zcat $qarchive|";
5419         } elsif ($comp eq 'lzop') {
5420             $uncomp = "lzop -d -c $qarchive|";
5421         } else {
5422             die "unknown compression method '$comp'\n";
5423         }
5424
5425     }
5426
5427     my $tmpdir = "/var/tmp/vzdumptmp$$";
5428     rmtree $tmpdir;
5429
5430     # disable interrupts (always do cleanups)
5431     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5432         warn "got interrupt - ignored\n";
5433     };
5434
5435     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5436     POSIX::mkfifo($mapfifo, 0600);
5437     my $fifofh;
5438
5439     my $openfifo = sub {
5440         open($fifofh, '>', $mapfifo) || die $!;
5441     };
5442
5443     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5444
5445     my $oldtimeout;
5446     my $timeout = 5;
5447
5448     my $devinfo = {};
5449
5450     my $rpcenv = PVE::RPCEnvironment::get();
5451
5452     my $conffile = PVE::QemuConfig->config_file($vmid);
5453     my $tmpfn = "$conffile.$$.tmp";
5454
5455     # Note: $oldconf is undef if VM does not exists
5456     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5457     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5458
5459     my $print_devmap = sub {
5460         my $virtdev_hash = {};
5461
5462         my $cfgfn = "$tmpdir/qemu-server.conf";
5463
5464         # we can read the config - that is already extracted
5465         my $fh = IO::File->new($cfgfn, "r") ||
5466             "unable to read qemu-server.conf - $!\n";
5467
5468         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5469         if (-f $fwcfgfn) {
5470             my $pve_firewall_dir = '/etc/pve/firewall';
5471             mkdir $pve_firewall_dir; # make sure the dir exists
5472             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5473         }
5474
5475         while (defined(my $line = <$fh>)) {
5476             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5477                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5478                 die "archive does not contain data for drive '$virtdev'\n"
5479                     if !$devinfo->{$devname};
5480                 if (defined($opts->{storage})) {
5481                     $storeid = $opts->{storage} || 'local';
5482                 } elsif (!$storeid) {
5483                     $storeid = 'local';
5484                 }
5485                 $format = 'raw' if !$format;
5486                 $devinfo->{$devname}->{devname} = $devname;
5487                 $devinfo->{$devname}->{virtdev} = $virtdev;
5488                 $devinfo->{$devname}->{format} = $format;
5489                 $devinfo->{$devname}->{storeid} = $storeid;
5490
5491                 # check permission on storage
5492                 my $pool = $opts->{pool}; # todo: do we need that?
5493                 if ($user ne 'root@pam') {
5494                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5495                 }
5496
5497                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5498             }
5499         }
5500
5501         foreach my $devname (keys %$devinfo) {
5502             die "found no device mapping information for device '$devname'\n"
5503                 if !$devinfo->{$devname}->{virtdev};
5504         }
5505
5506         my $cfg = PVE::Storage::config();
5507
5508         # create empty/temp config
5509         if ($oldconf) {
5510             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5511             foreach_drive($oldconf, sub {
5512                 my ($ds, $drive) = @_;
5513
5514                 return if drive_is_cdrom($drive);
5515
5516                 my $volid = $drive->{file};
5517
5518                 return if !$volid || $volid =~ m|^/|;
5519
5520                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5521                 return if !$path || !$owner || ($owner != $vmid);
5522
5523                 # Note: only delete disk we want to restore
5524                 # other volumes will become unused
5525                 if ($virtdev_hash->{$ds}) {
5526                     PVE::Storage::vdisk_free($cfg, $volid);
5527                 }
5528             });
5529
5530             # delete vmstate files
5531             # since after the restore we have no snapshots anymore
5532             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5533                 my $snap = $oldconf->{snapshots}->{$snapname};
5534                 if ($snap->{vmstate}) {
5535                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5536                     if (my $err = $@) {
5537                         warn $err;
5538                     }
5539                 }
5540             }
5541         }
5542
5543         my $map = {};
5544         foreach my $virtdev (sort keys %$virtdev_hash) {
5545             my $d = $virtdev_hash->{$virtdev};
5546             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5547             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5548
5549             # test if requested format is supported
5550             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5551             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5552             $d->{format} = $defFormat if !$supported;
5553
5554             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5555                                                   $d->{format}, undef, $alloc_size);
5556             print STDERR "new volume ID is '$volid'\n";
5557             $d->{volid} = $volid;
5558             my $path = PVE::Storage::path($cfg, $volid);
5559
5560             PVE::Storage::activate_volumes($cfg,[$volid]);
5561
5562             my $write_zeros = 1;
5563             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5564                 $write_zeros = 0;
5565             }
5566
5567             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5568
5569             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5570             $map->{$virtdev} = $volid;
5571         }
5572
5573         $fh->seek(0, 0) || die "seek failed - $!\n";
5574
5575         my $outfd = new IO::File ($tmpfn, "w") ||
5576             die "unable to write config for VM $vmid\n";
5577
5578         my $cookie = { netcount => 0 };
5579         while (defined(my $line = <$fh>)) {
5580             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5581         }
5582
5583         $fh->close();
5584         $outfd->close();
5585     };
5586
5587     eval {
5588         # enable interrupts
5589         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5590             die "interrupted by signal\n";
5591         };
5592         local $SIG{ALRM} = sub { die "got timeout\n"; };
5593
5594         $oldtimeout = alarm($timeout);
5595
5596         my $parser = sub {
5597             my $line = shift;
5598
5599             print "$line\n";
5600
5601             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5602                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5603                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5604             } elsif ($line =~ m/^CTIME: /) {
5605                 # we correctly received the vma config, so we can disable
5606                 # the timeout now for disk allocation (set to 10 minutes, so
5607                 # that we always timeout if something goes wrong)
5608                 alarm(600);
5609                 &$print_devmap();
5610                 print $fifofh "done\n";
5611                 my $tmp = $oldtimeout || 0;
5612                 $oldtimeout = undef;
5613                 alarm($tmp);
5614                 close($fifofh);
5615             }
5616         };
5617
5618         print "restore vma archive: $cmd\n";
5619         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5620     };
5621     my $err = $@;
5622
5623     alarm($oldtimeout) if $oldtimeout;
5624
5625     my $vollist = [];
5626     foreach my $devname (keys %$devinfo) {
5627         my $volid = $devinfo->{$devname}->{volid};
5628         push @$vollist, $volid if $volid;
5629     }
5630
5631     my $cfg = PVE::Storage::config();
5632     PVE::Storage::deactivate_volumes($cfg, $vollist);
5633
5634     unlink $mapfifo;
5635
5636     if ($err) {
5637         rmtree $tmpdir;
5638         unlink $tmpfn;
5639
5640         foreach my $devname (keys %$devinfo) {
5641             my $volid = $devinfo->{$devname}->{volid};
5642             next if !$volid;
5643             eval {
5644                 if ($volid =~ m|^/|) {
5645                     unlink $volid || die 'unlink failed\n';
5646                 } else {
5647                     PVE::Storage::vdisk_free($cfg, $volid);
5648                 }
5649                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5650             };
5651             print STDERR "unable to cleanup '$volid' - $@" if $@;
5652         }
5653         die $err;
5654     }
5655
5656     rmtree $tmpdir;
5657
5658     rename($tmpfn, $conffile) ||
5659         die "unable to commit configuration file '$conffile'\n";
5660
5661     PVE::Cluster::cfs_update(); # make sure we read new file
5662
5663     eval { rescan($vmid, 1); };
5664     warn $@ if $@;
5665 }
5666
5667 sub restore_tar_archive {
5668     my ($archive, $vmid, $user, $opts) = @_;
5669
5670     if ($archive ne '-') {
5671         my $firstfile = tar_archive_read_firstfile($archive);
5672         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5673             if $firstfile ne 'qemu-server.conf';
5674     }
5675
5676     my $storecfg = PVE::Storage::config();
5677
5678     # destroy existing data - keep empty config
5679     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5680     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5681
5682     my $tocmd = "/usr/lib/qemu-server/qmextract";
5683
5684     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5685     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5686     $tocmd .= ' --prealloc' if $opts->{prealloc};
5687     $tocmd .= ' --info' if $opts->{info};
5688
5689     # tar option "xf" does not autodetect compression when read from STDIN,
5690     # so we pipe to zcat
5691     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5692         PVE::Tools::shellquote("--to-command=$tocmd");
5693
5694     my $tmpdir = "/var/tmp/vzdumptmp$$";
5695     mkpath $tmpdir;
5696
5697     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5698     local $ENV{VZDUMP_VMID} = $vmid;
5699     local $ENV{VZDUMP_USER} = $user;
5700
5701     my $conffile = PVE::QemuConfig->config_file($vmid);
5702     my $tmpfn = "$conffile.$$.tmp";
5703
5704     # disable interrupts (always do cleanups)
5705     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5706         print STDERR "got interrupt - ignored\n";
5707     };
5708
5709     eval {
5710         # enable interrupts
5711         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5712             die "interrupted by signal\n";
5713         };
5714
5715         if ($archive eq '-') {
5716             print "extracting archive from STDIN\n";
5717             run_command($cmd, input => "<&STDIN");
5718         } else {
5719             print "extracting archive '$archive'\n";
5720             run_command($cmd);
5721         }
5722
5723         return if $opts->{info};
5724
5725         # read new mapping
5726         my $map = {};
5727         my $statfile = "$tmpdir/qmrestore.stat";
5728         if (my $fd = IO::File->new($statfile, "r")) {
5729             while (defined (my $line = <$fd>)) {
5730                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5731                     $map->{$1} = $2 if $1;
5732                 } else {
5733                     print STDERR "unable to parse line in statfile - $line\n";
5734                 }
5735             }
5736             $fd->close();
5737         }
5738
5739         my $confsrc = "$tmpdir/qemu-server.conf";
5740
5741         my $srcfd = new IO::File($confsrc, "r") ||
5742             die "unable to open file '$confsrc'\n";
5743
5744         my $outfd = new IO::File ($tmpfn, "w") ||
5745             die "unable to write config for VM $vmid\n";
5746
5747         my $cookie = { netcount => 0 };
5748         while (defined (my $line = <$srcfd>)) {
5749             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5750         }
5751
5752         $srcfd->close();
5753         $outfd->close();
5754     };
5755     my $err = $@;
5756
5757     if ($err) {
5758
5759         unlink $tmpfn;
5760
5761         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5762
5763         die $err;
5764     }
5765
5766     rmtree $tmpdir;
5767
5768     rename $tmpfn, $conffile ||
5769         die "unable to commit configuration file '$conffile'\n";
5770
5771     PVE::Cluster::cfs_update(); # make sure we read new file
5772
5773     eval { rescan($vmid, 1); };
5774     warn $@ if $@;
5775 };
5776
5777 sub foreach_writable_storage {
5778     my ($conf, $func) = @_;
5779
5780     my $sidhash = {};
5781
5782     foreach my $ds (keys %$conf) {
5783         next if !is_valid_drivename($ds);
5784
5785         my $drive = parse_drive($ds, $conf->{$ds});
5786         next if !$drive;
5787         next if drive_is_cdrom($drive);
5788
5789         my $volid = $drive->{file};
5790
5791         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5792         $sidhash->{$sid} = $sid if $sid;
5793     }
5794
5795     foreach my $sid (sort keys %$sidhash) {
5796         &$func($sid);
5797     }
5798 }
5799
5800 sub do_snapshots_with_qemu {
5801     my ($storecfg, $volid) = @_;
5802
5803     my $storage_name = PVE::Storage::parse_volume_id($volid);
5804
5805     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5806         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5807         return 1;
5808     }
5809
5810     if ($volid =~ m/\.(qcow2|qed)$/){
5811         return 1;
5812     }
5813
5814     return undef;
5815 }
5816
5817 sub qga_check_running {
5818     my ($vmid) = @_;
5819
5820     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5821     if ($@) {
5822         warn "Qemu Guest Agent are not running - $@";
5823         return 0;
5824     }
5825     return 1;
5826 }
5827
5828 sub template_create {
5829     my ($vmid, $conf, $disk) = @_;
5830
5831     my $storecfg = PVE::Storage::config();
5832
5833     foreach_drive($conf, sub {
5834         my ($ds, $drive) = @_;
5835
5836         return if drive_is_cdrom($drive);
5837         return if $disk && $ds ne $disk;
5838
5839         my $volid = $drive->{file};
5840         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5841
5842         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5843         $drive->{file} = $voliddst;
5844         $conf->{$ds} = print_drive($vmid, $drive);
5845         PVE::QemuConfig->write_config($vmid, $conf);
5846     });
5847 }
5848
5849 sub qemu_img_convert {
5850     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5851
5852     my $storecfg = PVE::Storage::config();
5853     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5854     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5855
5856     if ($src_storeid && $dst_storeid) {
5857
5858         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5859
5860         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5861         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5862
5863         my $src_format = qemu_img_format($src_scfg, $src_volname);
5864         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5865
5866         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5867         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5868
5869         my $cmd = [];
5870         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5871         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5872         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5873         if ($is_zero_initialized) {
5874             push @$cmd, "zeroinit:$dst_path";
5875         } else {
5876             push @$cmd, $dst_path;
5877         }
5878
5879         my $parser = sub {
5880             my $line = shift;
5881             if($line =~ m/\((\S+)\/100\%\)/){
5882                 my $percent = $1;
5883                 my $transferred = int($size * $percent / 100);
5884                 my $remaining = $size - $transferred;
5885
5886                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5887             }
5888
5889         };
5890
5891         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5892         my $err = $@;
5893         die "copy failed: $err" if $err;
5894     }
5895 }
5896
5897 sub qemu_img_format {
5898     my ($scfg, $volname) = @_;
5899
5900     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5901         return $1;
5902     } else {
5903         return "raw";
5904     }
5905 }
5906
5907 sub qemu_drive_mirror {
5908     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5909
5910     my $storecfg = PVE::Storage::config();
5911     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5912
5913     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5914
5915     my $format = qemu_img_format($dst_scfg, $dst_volname);
5916
5917     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5918
5919     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5920
5921     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5922     $opts->{format} = $format if $format;
5923
5924     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5925
5926     my $finish_job = sub {
5927         while (1) {
5928             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5929             my $stat = @$stats[0];
5930             last if !$stat;
5931             sleep 1;
5932         }
5933     };
5934
5935     eval {
5936     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5937         while (1) {
5938             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5939             my $stat = @$stats[0];
5940             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5941             die "error job is not mirroring" if $stat->{type} ne "mirror";
5942
5943             my $busy = $stat->{busy};
5944             my $ready = $stat->{ready};
5945
5946             if (my $total = $stat->{len}) {
5947                 my $transferred = $stat->{offset} || 0;
5948                 my $remaining = $total - $transferred;
5949                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5950
5951                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5952             }
5953
5954
5955             if ($stat->{ready} eq 'true') {
5956
5957                 last if $vmiddst != $vmid;
5958
5959                 # try to switch the disk if source and destination are on the same guest
5960                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5961                 if (!$@) {
5962                     &$finish_job();
5963                     last;
5964                 }
5965                 die $@ if $@ !~ m/cannot be completed/;
5966             }
5967             sleep 1;
5968         }
5969
5970
5971     };
5972     my $err = $@;
5973
5974     my $cancel_job = sub {
5975         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5976         &$finish_job();
5977     };
5978
5979     if ($err) {
5980         eval { &$cancel_job(); };
5981         die "mirroring error: $err";
5982     }
5983
5984     if ($vmiddst != $vmid) {
5985         # if we clone a disk for a new target vm, we don't switch the disk
5986         &$cancel_job(); # so we call block-job-cancel
5987     }
5988 }
5989
5990 sub clone_disk {
5991     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5992         $newvmid, $storage, $format, $full, $newvollist) = @_;
5993
5994     my $newvolid;
5995
5996     if (!$full) {
5997         print "create linked clone of drive $drivename ($drive->{file})\n";
5998         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5999         push @$newvollist, $newvolid;
6000     } else {
6001         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6002         $storeid = $storage if $storage;
6003
6004         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6005         if (!$format) {
6006             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6007             $format = qemu_img_format($scfg, $volname);
6008         }
6009
6010         # test if requested format is supported - else use default
6011         my $supported = grep { $_ eq $format } @$validFormats;
6012         $format = $defFormat if !$supported;
6013
6014         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6015
6016         print "create full clone of drive $drivename ($drive->{file})\n";
6017         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6018         push @$newvollist, $newvolid;
6019
6020         PVE::Storage::activate_volumes($storecfg, $newvollist);
6021
6022         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6023         if (!$running || $snapname) {
6024             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6025         } else {
6026             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
6027         }
6028     }
6029
6030     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6031
6032     my $disk = $drive;
6033     $disk->{format} = undef;
6034     $disk->{file} = $newvolid;
6035     $disk->{size} = $size;
6036
6037     return $disk;
6038 }
6039
6040 # this only works if VM is running
6041 sub get_current_qemu_machine {
6042     my ($vmid) = @_;
6043
6044     my $cmd = { execute => 'query-machines', arguments => {} };
6045     my $res = vm_qmp_command($vmid, $cmd);
6046
6047     my ($current, $default);
6048     foreach my $e (@$res) {
6049         $default = $e->{name} if $e->{'is-default'};
6050         $current = $e->{name} if $e->{'is-current'};
6051     }
6052
6053     # fallback to the default machine if current is not supported by qemu
6054     return $current || $default || 'pc';
6055 }
6056
6057 sub qemu_machine_feature_enabled {
6058     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6059
6060     my $current_major;
6061     my $current_minor;
6062
6063     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6064
6065         $current_major = $3;
6066         $current_minor = $4;
6067
6068     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6069
6070         $current_major = $1;
6071         $current_minor = $2;
6072     }
6073
6074     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6075
6076
6077 }
6078
6079 sub qemu_machine_pxe {
6080     my ($vmid, $conf, $machine) = @_;
6081
6082     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6083
6084     foreach my $opt (keys %$conf) {
6085         next if $opt !~ m/^net(\d+)$/;
6086         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6087         next if !$net;
6088         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6089         return $machine.".pxe" if $romfile =~ m/pxe/;
6090         last;
6091     }
6092
6093     return $machine;
6094 }
6095
6096 sub qemu_use_old_bios_files {
6097     my ($machine_type) = @_;
6098
6099     return if !$machine_type;
6100
6101     my $use_old_bios_files = undef;
6102
6103     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6104         $machine_type = $1;
6105         $use_old_bios_files = 1;
6106     } else {
6107         my $kvmver = kvm_user_version();
6108         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6109         # load new efi bios files on migration. So this hack is required to allow
6110         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6111         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6112         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6113     }
6114
6115     return ($use_old_bios_files, $machine_type);
6116 }
6117
6118 sub lspci {
6119
6120     my $devices = {};
6121
6122     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6123             my (undef, $id, $function) = @_;
6124             my $res = { id => $id, function => $function};
6125             push @{$devices->{$id}}, $res;
6126     });
6127
6128     return $devices;
6129 }
6130
6131 sub vm_iothreads_list {
6132     my ($vmid) = @_;
6133
6134     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6135
6136     my $iothreads = {};
6137     foreach my $iothread (@$res) {
6138         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6139     }
6140
6141     return $iothreads;
6142 }
6143
6144 sub scsihw_infos {
6145     my ($conf, $drive) = @_;
6146
6147     my $maxdev = 0;
6148
6149     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6150         $maxdev = 7;
6151     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6152         $maxdev = 1;
6153     } else {
6154         $maxdev = 256;
6155     }
6156
6157     my $controller = int($drive->{index} / $maxdev);
6158     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6159
6160     return ($maxdev, $controller, $controller_prefix);
6161 }
6162
6163 # bash completion helper
6164
6165 sub complete_backup_archives {
6166     my ($cmdname, $pname, $cvalue) = @_;
6167
6168     my $cfg = PVE::Storage::config();
6169
6170     my $storeid;
6171
6172     if ($cvalue =~ m/^([^:]+):/) {
6173         $storeid = $1;
6174     }
6175
6176     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6177
6178     my $res = [];
6179     foreach my $id (keys %$data) {
6180         foreach my $item (@{$data->{$id}}) {
6181             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6182             push @$res, $item->{volid} if defined($item->{volid});
6183         }
6184     }
6185
6186     return $res;
6187 }
6188
6189 my $complete_vmid_full = sub {
6190     my ($running) = @_;
6191
6192     my $idlist = vmstatus();
6193
6194     my $res = [];
6195
6196     foreach my $id (keys %$idlist) {
6197         my $d = $idlist->{$id};
6198         if (defined($running)) {
6199             next if $d->{template};
6200             next if $running && $d->{status} ne 'running';
6201             next if !$running && $d->{status} eq 'running';
6202         }
6203         push @$res, $id;
6204
6205     }
6206     return $res;
6207 };
6208
6209 sub complete_vmid {
6210     return &$complete_vmid_full();
6211 }
6212
6213 sub complete_vmid_stopped {
6214     return &$complete_vmid_full(0);
6215 }
6216
6217 sub complete_vmid_running {
6218     return &$complete_vmid_full(1);
6219 }
6220
6221 sub complete_storage {
6222
6223     my $cfg = PVE::Storage::config();
6224     my $ids = $cfg->{ids};
6225
6226     my $res = [];
6227     foreach my $sid (keys %$ids) {
6228         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6229         next if !$ids->{$sid}->{content}->{images};
6230         push @$res, $sid;
6231     }
6232
6233     return $res;
6234 }
6235
6236 1;