PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35 use URI::Escape;
  36
  37 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  38
  39 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  40
  41 # Note about locking: we use flock on the config file protect
  42 # against concurent actions.
  43 # Aditionaly, we have a 'lock' setting in the config file. This
  44 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  45 # allowed when such lock is set. But you can ignore this kind of
  46 # lock with the --skiplock flag.
  47
  48 cfs_register_file('/qemu-server/',
  49                   \&parse_vm_config,
  50                   \&write_vm_config);
  51
  52 PVE::JSONSchema::register_standard_option('skiplock', {
  53     description => "Ignore locks - only root is allowed to use this option.",
  54     type => 'boolean',
  55     optional => 1,
  56 });
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 #no warnings 'redefine';
  72
  73 sub cgroups_write {
  74    my ($controller, $vmid, $option, $value) = @_;
  75
  76    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  77    PVE::ProcFSTools::write_proc_entry($path, $value);
  78
  79 }
  80
  81 my $nodename = PVE::INotify::nodename();
  82
  83 mkdir "/etc/pve/nodes/$nodename";
  84 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  85 mkdir $confdir;
  86
  87 my $var_run_tmpdir = "/var/run/qemu-server";
  88 mkdir $var_run_tmpdir;
  89
  90 my $lock_dir = "/var/lock/qemu-server";
  91 mkdir $lock_dir;
  92
  93 my $pcisysfs = "/sys/bus/pci";
  94
  95 my $cpu_fmt = {
  96     cputype => {
  97         description => "Emulated CPU type.",
  98         type => 'string',
  99         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 100         format_description => 'cputype',
 101         default => 'kvm64',
 102         default_key => 1,
 103     },
 104     hidden => {
 105         description => "Do not identify as a KVM virtual machine.",
 106         type => 'boolean',
 107         optional => 1,
 108         default => 0
 109     },
 110 };
 111
 112 my $watchdog_fmt = {
 113     model => {
 114         default_key => 1,
 115         type => 'string',
 116         enum => [qw(i6300esb ib700)],
 117         description => "Watchdog type to emulate.",
 118         default => 'i6300esb',
 119         optional => 1,
 120     },
 121     action => {
 122         type => 'string',
 123         enum => [qw(reset shutdown poweroff pause debug none)],
 124         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 125         optional => 1,
 126     },
 127 };
 128 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 129
 130 my $confdesc = {
 131     onboot => {
 132         optional => 1,
 133         type => 'boolean',
 134         description => "Specifies whether a VM will be started during system bootup.",
 135         default => 0,
 136     },
 137     autostart => {
 138         optional => 1,
 139         type => 'boolean',
 140         description => "Automatic restart after crash (currently ignored).",
 141         default => 0,
 142     },
 143     hotplug => {
 144         optional => 1,
 145         type => 'string', format => 'pve-hotplug-features',
 146         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 147         default => 'network,disk,usb',
 148     },
 149     reboot => {
 150         optional => 1,
 151         type => 'boolean',
 152         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 153         default => 1,
 154     },
 155     lock => {
 156         optional => 1,
 157         type => 'string',
 158         description => "Lock/unlock the VM.",
 159         enum => [qw(migrate backup snapshot rollback)],
 160     },
 161     cpulimit => {
 162         optional => 1,
 163         type => 'number',
 164         description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 165         minimum => 0,
 166         maximum => 128,
 167         default => 0,
 168     },
 169     cpuunits => {
 170         optional => 1,
 171         type => 'integer',
 172         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 173         minimum => 0,
 174         maximum => 500000,
 175         default => 1000,
 176     },
 177     memory => {
 178         optional => 1,
 179         type => 'integer',
 180         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 181         minimum => 16,
 182         default => 512,
 183     },
 184     balloon => {
 185         optional => 1,
 186         type => 'integer',
 187         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 188         minimum => 0,
 189     },
 190     shares => {
 191         optional => 1,
 192         type => 'integer',
 193         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 194         minimum => 0,
 195         maximum => 50000,
 196         default => 1000,
 197     },
 198     keyboard => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 202         enum => PVE::Tools::kvmkeymaplist(),
 203         default => 'en-us',
 204     },
 205     name => {
 206         optional => 1,
 207         type => 'string', format => 'dns-name',
 208         description => "Set a name for the VM. Only used on the configuration web interface.",
 209     },
 210     scsihw => {
 211         optional => 1,
 212         type => 'string',
 213         description => "scsi controller model",
 214         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 215         default => 'lsi',
 216     },
 217     description => {
 218         optional => 1,
 219         type => 'string',
 220         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 221     },
 222     ostype => {
 223         optional => 1,
 224         type => 'string',
 225         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 226         description => <<EODESC,
 227 Used to enable special optimization/features for specific
 228 operating systems:
 229
 230 other  => unspecified OS
 231 wxp    => Microsoft Windows XP
 232 w2k    => Microsoft Windows 2000
 233 w2k3   => Microsoft Windows 2003
 234 w2k8   => Microsoft Windows 2008
 235 wvista => Microsoft Windows Vista
 236 win7   => Microsoft Windows 7
 237 win8   => Microsoft Windows 8/2012
 238 l24    => Linux 2.4 Kernel
 239 l26    => Linux 2.6/3.X Kernel
 240 solaris => solaris/opensolaris/openindiania kernel
 241
 242 other|l24|l26|solaris                       ... no special behaviour
 243 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 244 EODESC
 245     },
 246     boot => {
 247         optional => 1,
 248         type => 'string',
 249         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 250         pattern => '[acdn]{1,4}',
 251         default => 'cdn',
 252     },
 253     bootdisk => {
 254         optional => 1,
 255         type => 'string', format => 'pve-qm-bootdisk',
 256         description => "Enable booting from specified disk.",
 257         pattern => '(ide|sata|scsi|virtio)\d+',
 258     },
 259     smp => {
 260         optional => 1,
 261         type => 'integer',
 262         description => "The number of CPUs. Please use option -sockets instead.",
 263         minimum => 1,
 264         default => 1,
 265     },
 266     sockets => {
 267         optional => 1,
 268         type => 'integer',
 269         description => "The number of CPU sockets.",
 270         minimum => 1,
 271         default => 1,
 272     },
 273     cores => {
 274         optional => 1,
 275         type => 'integer',
 276         description => "The number of cores per socket.",
 277         minimum => 1,
 278         default => 1,
 279     },
 280     numa => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Enable/disable NUMA.",
 284         default => 0,
 285     },
 286     vcpus => {
 287         optional => 1,
 288         type => 'integer',
 289         description => "Number of hotplugged vcpus.",
 290         minimum => 1,
 291         default => 0,
 292     },
 293     acpi => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Enable/disable ACPI.",
 297         default => 1,
 298     },
 299     agent => {
 300         optional => 1,
 301         type => 'boolean',
 302         description => "Enable/disable Qemu GuestAgent.",
 303         default => 0,
 304     },
 305     kvm => {
 306         optional => 1,
 307         type => 'boolean',
 308         description => "Enable/disable KVM hardware virtualization.",
 309         default => 1,
 310     },
 311     tdf => {
 312         optional => 1,
 313         type => 'boolean',
 314         description => "Enable/disable time drift fix.",
 315         default => 0,
 316     },
 317     localtime => {
 318         optional => 1,
 319         type => 'boolean',
 320         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 321     },
 322     freeze => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 326     },
 327     vga => {
 328         optional => 1,
 329         type => 'string',
 330         description => "Select the VGA type. If you want to use high resolution" .
 331             " modes (>= 1280x1024x16) then you should use the options " .
 332             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 333             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 334             "display sever. For win* OS you can select how many independent " .
 335             "displays you want, Linux guests can add displays them self. " .
 336             "You can also run without any graphic card, using a serial device" .
 337             " as terminal.",
 338         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 339     },
 340     watchdog => {
 341         optional => 1,
 342         type => 'string', format => 'pve-qm-watchdog',
 343         description => "Create a virtual hardware watchdog device. Once enabled" .
 344             " (by a guest action), the watchdog must be periodically polled " .
 345             "by an agent inside the guest or else the watchdog will reset " .
 346             "the guest (or execute the respective action specified)",
 347     },
 348     startdate => {
 349         optional => 1,
 350         type => 'string',
 351         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 352         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 353         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 354         default => 'now',
 355     },
 356     startup =>  get_standard_option('pve-startup-order'),
 357     template => {
 358         optional => 1,
 359         type => 'boolean',
 360         description => "Enable/disable Template.",
 361         default => 0,
 362     },
 363     args => {
 364         optional => 1,
 365         type => 'string',
 366         description => <<EODESCR,
 367 Arbitrary arguments passed to kvm, for example:
 368
 369 args: -no-reboot -no-hpet
 370
 371 NOTE: this option is for experts only.
 372 EODESCR
 373     },
 374     tablet => {
 375         optional => 1,
 376         type => 'boolean',
 377         default => 1,
 378         description => "Enable/disable the USB tablet device. This device is " .
 379             "usually needed to allow absolute mouse positioning with VNC. " .
 380             "Else the mouse runs out of sync with normal VNC clients. " .
 381             "If you're running lots of console-only guests on one host, " .
 382             "you may consider disabling this to save some context switches. " .
 383             "This is turned off by default if you use spice (-vga=qxl).",
 384     },
 385     migrate_speed => {
 386         optional => 1,
 387         type => 'integer',
 388         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 389         minimum => 0,
 390         default => 0,
 391     },
 392     migrate_downtime => {
 393         optional => 1,
 394         type => 'number',
 395         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 396         minimum => 0,
 397         default => 0.1,
 398     },
 399     cdrom => {
 400         optional => 1,
 401         type => 'string', format => 'pve-qm-drive',
 402         typetext => 'volume',
 403         description => "This is an alias for option -ide2",
 404     },
 405     cpu => {
 406         optional => 1,
 407         description => "Emulated CPU type.",
 408         type => 'string',
 409         format => $cpu_fmt,
 410     },
 411     parent => get_standard_option('pve-snapshot-name', {
 412         optional => 1,
 413         description => "Parent snapshot name. This is used internally, and should not be modified.",
 414     }),
 415     snaptime => {
 416         optional => 1,
 417         description => "Timestamp for snapshots.",
 418         type => 'integer',
 419         minimum => 0,
 420     },
 421     vmstate => {
 422         optional => 1,
 423         type => 'string', format => 'pve-volume-id',
 424         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 425     },
 426     machine => {
 427         description => "Specific the Qemu machine type.",
 428         type => 'string',
 429         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 430         maxLength => 40,
 431         optional => 1,
 432     },
 433     smbios1 => {
 434         description => "Specify SMBIOS type 1 fields.",
 435         type => 'string', format => 'pve-qm-smbios1',
 436         maxLength => 256,
 437         optional => 1,
 438     },
 439     protection => {
 440         optional => 1,
 441         type => 'boolean',
 442         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 443         default => 0,
 444     },
 445     bios => {
 446         optional => 1,
 447         type => 'string',
 448         enum => [ qw(seabios ovmf) ],
 449         description => "Select BIOS implementation.",
 450         default => 'seabios',
 451     },
 452 };
 453
 454 # what about other qemu settings ?
 455 #cpu => 'string',
 456 #machine => 'string',
 457 #fda => 'file',
 458 #fdb => 'file',
 459 #mtdblock => 'file',
 460 #sd => 'file',
 461 #pflash => 'file',
 462 #snapshot => 'bool',
 463 #bootp => 'file',
 464 ##tftp => 'dir',
 465 ##smb => 'dir',
 466 #kernel => 'file',
 467 #append => 'string',
 468 #initrd => 'file',
 469 ##soundhw => 'string',
 470
 471 while (my ($k, $v) = each %$confdesc) {
 472     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 473 }
 474
 475 my $MAX_IDE_DISKS = 4;
 476 my $MAX_SCSI_DISKS = 14;
 477 my $MAX_VIRTIO_DISKS = 16;
 478 my $MAX_SATA_DISKS = 6;
 479 my $MAX_USB_DEVICES = 5;
 480 my $MAX_NETS = 32;
 481 my $MAX_UNUSED_DISKS = 8;
 482 my $MAX_HOSTPCI_DEVICES = 4;
 483 my $MAX_SERIAL_PORTS = 4;
 484 my $MAX_PARALLEL_PORTS = 3;
 485 my $MAX_NUMA = 8;
 486 my $MAX_MEM = 4194304;
 487 my $STATICMEM = 1024;
 488
 489 my $numa_fmt = {
 490     cpus => {
 491         type => "string",
 492         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 493         description => "CPUs accessing this numa node.",
 494         format_description => "id[-id];...",
 495     },
 496     memory => {
 497         type => "number",
 498         description => "Amount of memory this numa node provides.",
 499         format_description => "mb",
 500         optional => 1,
 501     },
 502     hostnodes => {
 503         type => "string",
 504         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 505         description => "host numa nodes to use",
 506         format_description => "id[-id];...",
 507         optional => 1,
 508     },
 509     policy => {
 510         type => 'string',
 511         enum => [qw(preferred bind interleave)],
 512         format_description => 'preferred|bind|interleave',
 513         description => "numa allocation policy.",
 514         optional => 1,
 515     },
 516 };
 517 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 518 my $numadesc = {
 519     optional => 1,
 520     type => 'string', format => $numa_fmt,
 521     description => "numa topology",
 522 };
 523 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 524
 525 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 526     $confdesc->{"numa$i"} = $numadesc;
 527 }
 528
 529 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 530                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 531                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 532 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 533
 534 my $net_fmt = {
 535     macaddr => {
 536         type => 'string',
 537         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 538         description => "MAC address",
 539         format_description => "XX:XX:XX:XX:XX:XX",
 540         optional => 1,
 541     },
 542     model => { alias => 'macaddr', default_key => 1 },
 543     (map { $_ => { group => 'model' } } @$nic_model_list),
 544     bridge => {
 545         type => 'string',
 546         description => 'Bridge to attach the network device to.',
 547         format_description => 'bridge',
 548         optional => 1,
 549     },
 550     queues => {
 551         type => 'integer',
 552         minimum => 0, maximum => 16,
 553         description => 'Number of packet queues to be used on the device.',
 554         format_description => 'number',
 555         optional => 1,
 556     },
 557     rate => {
 558         type => 'number',
 559         minimum => 0,
 560         description => 'Rate limit in mbps as floating point number.',
 561         format_description => 'mbps',
 562         optional => 1,
 563     },
 564     tag => {
 565         type => 'integer',
 566         minimum => 2, maximum => 4094,
 567         description => 'VLAN tag to apply to packets on this interface.',
 568         format_description => 'vlanid',
 569         optional => 1,
 570     },
 571     trunks => {
 572         type => 'string',
 573         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 574         description => 'VLAN trunks to pass through this interface.',
 575         format_description => 'id;id...',
 576         optional => 1,
 577     },
 578     firewall => {
 579         type => 'boolean',
 580         description => 'Whether this interface should be protected by the firewall.',
 581         format_description => '0|1',
 582         optional => 1,
 583     },
 584     link_down => {
 585         type => 'boolean',
 586         description => 'Whether this interface should be DISconnected (like pulling the plug).',
 587         format_description => '0|1',
 588         optional => 1,
 589     },
 590 };
 591 my $netdesc = {
 592     optional => 1,
 593     type => 'string', format => 'pve-qm-net',
 594     description => <<EODESCR,
 595 Specify network devices.
 596
 597 MODEL is one of: $nic_model_list_txt
 598
 599 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 600 automatically generated if not specified.
 601
 602 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 603
 604 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 605
 606 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 607
 608 10.0.2.2   Gateway
 609 10.0.2.3   DNS Server
 610 10.0.2.4   SMB Server
 611
 612 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 613
 614 EODESCR
 615 };
 616 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 617
 618 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 619     $confdesc->{"net$i"} = $netdesc;
 620 }
 621
 622 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 623 sub verify_volume_id_or_qm_path {
 624     my ($volid, $noerr) = @_;
 625
 626     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 627         return $volid;
 628     }
 629
 630     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 631     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 632     if ($@) {
 633         return undef if $noerr;
 634         die $@;
 635     }
 636     return $volid;
 637 }
 638
 639 my $drivename_hash;
 640
 641 my %drivedesc_base = (
 642     volume => { alias => 'file' },
 643     file => {
 644         type => 'string',
 645         format => 'pve-volume-id-or-qm-path',
 646         default_key => 1,
 647         format_description => 'volume',
 648         description => "The drive's backing volume.",
 649     },
 650     media => {
 651         type => 'string',
 652         format_description => 'cdrom|disk',
 653         enum => [qw(cdrom disk)],
 654         description => "The drive's media type.",
 655         default => 'disk',
 656         optional => 1
 657     },
 658     cyls => {
 659         type => 'integer',
 660         format_description => 'count',
 661         description => "Force the drive's physical geometry to have a specific cylinder count.",
 662         optional => 1
 663     },
 664     heads => {
 665         type => 'integer',
 666         format_description => 'count',
 667         description => "Force the drive's physical geometry to have a specific head count.",
 668         optional => 1
 669     },
 670     secs => {
 671         type => 'integer',
 672         format_description => 'count',
 673         description => "Force the drive's physical geometry to have a specific sector count.",
 674         optional => 1
 675     },
 676     trans => {
 677         type => 'string',
 678         format_description => 'none|lba|auto',
 679         enum => [qw(none lba auto)],
 680         description => "Force disk geometry bios translation mode.",
 681         optional => 1,
 682     },
 683     snapshot => {
 684         type => 'boolean',
 685         format_description => 'on|off',
 686         description => "Whether the drive should be included when making snapshots.",
 687         optional => 1,
 688     },
 689     cache => {
 690         type => 'string',
 691         format_description => 'none|writethrough|writeback|unsafe|directsync',
 692         enum => [qw(none writethrough writeback unsafe directsync)],
 693         description => "The drive's cache mode",
 694         optional => 1,
 695     },
 696     format => {
 697         type => 'string',
 698         format_description => 'drive format',
 699         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 700         description => "The drive's backing file's data format.",
 701         optional => 1,
 702     },
 703     size => {
 704         type => 'string',
 705         format => 'disk-size',
 706         description => "Disk size. This is purely informational and has no effect.",
 707         optional => 1,
 708     },
 709     backup => {
 710         type => 'boolean',
 711         format_description => 'on|off',
 712         description => "Whether the drive should be included when making backups.",
 713         optional => 1,
 714     },
 715     werror => {
 716         type => 'string',
 717         format_description => 'enospc|ignore|report|stop',
 718         enum => [qw(enospc ignore report stop)],
 719         description => 'Write error action.',
 720         optional => 1,
 721     },
 722     aio => {
 723         type => 'string',
 724         format_description => 'native|threads',
 725         enum => [qw(native threads)],
 726         description => 'AIO type to use.',
 727         optional => 1,
 728     },
 729     discard => {
 730         type => 'string',
 731         format_description => 'ignore|on',
 732         enum => [qw(ignore on)],
 733         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 734         optional => 1,
 735     },
 736     detect_zeroes => {
 737         type => 'boolean',
 738         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 739         optional => 1,
 740     },
 741     serial => {
 742         type => 'string',
 743         format => 'urlencoded',
 744         format_description => 'serial',
 745         maxLength => 20*3, # *3 since it's %xx url enoded
 746         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 747         optional => 1,
 748     }
 749 );
 750
 751 my %rerror_fmt = (
 752     rerror => {
 753         type => 'string',
 754         format_description => 'ignore|report|stop',
 755         enum => [qw(ignore report stop)],
 756         description => 'Read error action.',
 757         optional => 1,
 758     },
 759 );
 760
 761 my %iothread_fmt = ( iothread => {
 762         type => 'boolean',
 763         format_description => 'off|on',
 764         description => "Whether to use iothreads for this drive",
 765         optional => 1,
 766 });
 767
 768 my %model_fmt = (
 769     model => {
 770         type => 'string',
 771         format => 'urlencoded',
 772         format_description => 'model',
 773         maxLength => 40*3, # *3 since it's %xx url enoded
 774         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 775         optional => 1,
 776     },
 777 );
 778
 779 my %queues_fmt = (
 780     queues => {
 781         type => 'integer',
 782         format_description => 'nbqueues',
 783         description => "Number of queues.",
 784         minimum => 2,
 785         optional => 1
 786     }
 787 );
 788
 789 my $add_throttle_desc = sub {
 790     my ($key, $type, $what, $size, $longsize) = @_;
 791     $drivedesc_base{$key} = {
 792         type => $type,
 793         format_description => $size,
 794         description => "Maximum $what speed in $longsize per second.",
 795         optional => 1,
 796     };
 797 };
 798 # throughput: (leaky bucket)
 799 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 800 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 801 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 802 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 803 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 804 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 805 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 806 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 807 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 808
 809 # pools: (pool of IO before throttling starts taking effect)
 810 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 811 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 812 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 813 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 814 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 815 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 816
 817 my $ide_fmt = {
 818     %drivedesc_base,
 819     %rerror_fmt,
 820     %model_fmt,
 821 };
 822
 823 my $idedesc = {
 824     optional => 1,
 825     type => 'string', format => $ide_fmt,
 826     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 827 };
 828 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 829
 830 my $scsi_fmt = {
 831     %drivedesc_base,
 832     %iothread_fmt,
 833     %queues_fmt,
 834 };
 835 my $scsidesc = {
 836     optional => 1,
 837     type => 'string', format => $scsi_fmt,
 838     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 839 };
 840 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 841
 842 my $sata_fmt = {
 843     %drivedesc_base,
 844     %rerror_fmt,
 845 };
 846 my $satadesc = {
 847     optional => 1,
 848     type => 'string', format => $sata_fmt,
 849     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 850 };
 851 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 852
 853 my $virtio_fmt = {
 854     %drivedesc_base,
 855     %iothread_fmt,
 856     %rerror_fmt,
 857 };
 858 my $virtiodesc = {
 859     optional => 1,
 860     type => 'string', format => $virtio_fmt,
 861     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 862 };
 863 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 864
 865 my $alldrive_fmt = {
 866     %drivedesc_base,
 867     %rerror_fmt,
 868     %iothread_fmt,
 869     %model_fmt,
 870     %queues_fmt,
 871 };
 872
 873 my $usb_fmt = {
 874     host => {
 875         default_key => 1,
 876         type => 'string', format => 'pve-qm-usb-device',
 877         format_description => 'HOSTUSBDEVICE|spice',
 878         description => 'The Host USB device or port or the value spice',
 879     },
 880     usb3 => {
 881         optional => 1,
 882         type => 'boolean',
 883         format_description => 'yes|no',
 884         description => 'Specifies whether if given host option is a USB3 device or port',
 885     },
 886 };
 887
 888 my $usbdesc = {
 889     optional => 1,
 890     type => 'string', format => $usb_fmt,
 891     description => <<EODESCR,
 892 Configure an USB device (n is 0 to 4). This can be used to
 893 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 894
 895 'bus-port(.port)*' (decimal numbers) or
 896 'vendor_id:product_id' (hexadeciaml numbers) or
 897 'spice'
 898
 899 You can use the 'lsusb -t' command to list existing usb devices.
 900
 901 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 902
 903 The value 'spice' can be used to add a usb redirection devices for spice.
 904
 905 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 906
 907 EODESCR
 908 };
 909 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 910
 911 # NOTE: the match-groups of this regex are used in parse_hostpci
 912 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 913 my $hostpci_fmt = {
 914     host => {
 915         default_key => 1,
 916         type => 'string',
 917         pattern => qr/$PCIRE(;$PCIRE)*/,
 918         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 919         description => "The PCI ID of a host's PCI device or a list of PCI virtual functions of the host.",
 920     },
 921     rombar => {
 922         type => 'boolean',
 923         optional => 1,
 924         default => 1,
 925     },
 926     pcie => {
 927         type => 'boolean',
 928         optional => 1,
 929         default => 0,
 930     },
 931     'x-vga' => {
 932         type => 'boolean',
 933         optional => 1,
 934         default => 0,
 935     },
 936 };
 937 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
 938
 939 my $hostpcidesc = {
 940         optional => 1,
 941         type => 'string', format => 'pve-qm-hostpci',
 942         description => <<EODESCR,
 943 Map host pci devices. HOSTPCIDEVICE syntax is:
 944
 945 'bus:dev.func' (hexadecimal numbers)
 946
 947 You can us the 'lspci' command to list existing pci devices.
 948
 949 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 950
 951 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 952
 953 Experimental: user reported problems with this option.
 954 EODESCR
 955 };
 956 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 957
 958 my $serialdesc = {
 959         optional => 1,
 960         type => 'string',
 961         pattern => '(/dev/.+|socket)',
 962         description =>  <<EODESCR,
 963 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 964
 965 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 966
 967 Experimental: user reported problems with this option.
 968 EODESCR
 969 };
 970
 971 my $paralleldesc= {
 972         optional => 1,
 973         type => 'string',
 974         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 975         description =>  <<EODESCR,
 976 Map host parallel devices (n is 0 to 2).
 977
 978 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 979
 980 Experimental: user reported problems with this option.
 981 EODESCR
 982 };
 983
 984 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 985     $confdesc->{"parallel$i"} = $paralleldesc;
 986 }
 987
 988 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 989     $confdesc->{"serial$i"} = $serialdesc;
 990 }
 991
 992 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 993     $confdesc->{"hostpci$i"} = $hostpcidesc;
 994 }
 995
 996 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 997     $drivename_hash->{"ide$i"} = 1;
 998     $confdesc->{"ide$i"} = $idedesc;
 999 }
1000
1001 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1002     $drivename_hash->{"sata$i"} = 1;
1003     $confdesc->{"sata$i"} = $satadesc;
1004 }
1005
1006 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1007     $drivename_hash->{"scsi$i"} = 1;
1008     $confdesc->{"scsi$i"} = $scsidesc ;
1009 }
1010
1011 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1012     $drivename_hash->{"virtio$i"} = 1;
1013     $confdesc->{"virtio$i"} = $virtiodesc;
1014 }
1015
1016 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1017     $confdesc->{"usb$i"} = $usbdesc;
1018 }
1019
1020 my $unuseddesc = {
1021     optional => 1,
1022     type => 'string', format => 'pve-volume-id',
1023     description => "Reference to unused volumes.",
1024 };
1025
1026 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1027     $confdesc->{"unused$i"} = $unuseddesc;
1028 }
1029
1030 my $kvm_api_version = 0;
1031
1032 sub kvm_version {
1033
1034     return $kvm_api_version if $kvm_api_version;
1035
1036     my $fh = IO::File->new("</dev/kvm") ||
1037         return 0;
1038
1039     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1040         $kvm_api_version = $v;
1041     }
1042
1043     $fh->close();
1044
1045     return  $kvm_api_version;
1046 }
1047
1048 my $kvm_user_version;
1049
1050 sub kvm_user_version {
1051
1052     return $kvm_user_version if $kvm_user_version;
1053
1054     $kvm_user_version = 'unknown';
1055
1056     my $code = sub {
1057         my $line = shift;
1058         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1059             $kvm_user_version = $2;
1060         }
1061     };
1062
1063     eval { run_command("kvm -version", outfunc => $code); };
1064     warn $@ if $@;
1065
1066     return $kvm_user_version;
1067
1068 }
1069
1070 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1071
1072 sub valid_drive_names {
1073     # order is important - used to autoselect boot disk
1074     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1075             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1076             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1077             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1078 }
1079
1080 sub is_valid_drivename {
1081     my $dev = shift;
1082
1083     return defined($drivename_hash->{$dev});
1084 }
1085
1086 sub option_exists {
1087     my $key = shift;
1088     return defined($confdesc->{$key});
1089 }
1090
1091 sub nic_models {
1092     return $nic_model_list;
1093 }
1094
1095 sub os_list_description {
1096
1097     return {
1098         other => 'Other',
1099         wxp => 'Windows XP',
1100         w2k => 'Windows 2000',
1101         w2k3 =>, 'Windows 2003',
1102         w2k8 => 'Windows 2008',
1103         wvista => 'Windows Vista',
1104         win7 => 'Windows 7',
1105         win8 => 'Windows 8/2012',
1106         l24 => 'Linux 2.4',
1107         l26 => 'Linux 2.6',
1108     };
1109 }
1110
1111 my $cdrom_path;
1112
1113 sub get_cdrom_path {
1114
1115     return  $cdrom_path if $cdrom_path;
1116
1117     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1118     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1119     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1120 }
1121
1122 sub get_iso_path {
1123     my ($storecfg, $vmid, $cdrom) = @_;
1124
1125     if ($cdrom eq 'cdrom') {
1126         return get_cdrom_path();
1127     } elsif ($cdrom eq 'none') {
1128         return '';
1129     } elsif ($cdrom =~ m|^/|) {
1130         return $cdrom;
1131     } else {
1132         return PVE::Storage::path($storecfg, $cdrom);
1133     }
1134 }
1135
1136 # try to convert old style file names to volume IDs
1137 sub filename_to_volume_id {
1138     my ($vmid, $file, $media) = @_;
1139
1140     if (!($file eq 'none' || $file eq 'cdrom' ||
1141           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1142
1143         return undef if $file =~ m|/|;
1144
1145         if ($media && $media eq 'cdrom') {
1146             $file = "local:iso/$file";
1147         } else {
1148             $file = "local:$vmid/$file";
1149         }
1150     }
1151
1152     return $file;
1153 }
1154
1155 sub verify_media_type {
1156     my ($opt, $vtype, $media) = @_;
1157
1158     return if !$media;
1159
1160     my $etype;
1161     if ($media eq 'disk') {
1162         $etype = 'images';
1163     } elsif ($media eq 'cdrom') {
1164         $etype = 'iso';
1165     } else {
1166         die "internal error";
1167     }
1168
1169     return if ($vtype eq $etype);
1170
1171     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1172 }
1173
1174 sub cleanup_drive_path {
1175     my ($opt, $storecfg, $drive) = @_;
1176
1177     # try to convert filesystem paths to volume IDs
1178
1179     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1180         ($drive->{file} !~ m|^/dev/.+|) &&
1181         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1182         ($drive->{file} !~ m/^\d+$/)) {
1183         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1184         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1185         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1186         verify_media_type($opt, $vtype, $drive->{media});
1187         $drive->{file} = $volid;
1188     }
1189
1190     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1191 }
1192
1193 sub parse_hotplug_features {
1194     my ($data) = @_;
1195
1196     my $res = {};
1197
1198     return $res if $data eq '0';
1199
1200     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1201
1202     foreach my $feature (PVE::Tools::split_list($data)) {
1203         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1204             $res->{$1} = 1;
1205         } else {
1206             warn "ignoring unknown hotplug feature '$feature'\n";
1207         }
1208     }
1209     return $res;
1210 }
1211
1212 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1213 sub pve_verify_hotplug_features {
1214     my ($value, $noerr) = @_;
1215
1216     return $value if parse_hotplug_features($value);
1217
1218     return undef if $noerr;
1219
1220     die "unable to parse hotplug option\n";
1221 }
1222
1223 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1224 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1225 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1226 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1227 #        [,iothread=on][,serial=serial][,model=model]
1228
1229 sub parse_drive {
1230     my ($key, $data) = @_;
1231
1232     my ($interface, $index);
1233
1234     if ($key =~ m/^([^\d]+)(\d+)$/) {
1235         $interface = $1;
1236         $index = $2;
1237     } else {
1238         return undef;
1239     }
1240
1241     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1242                                      : $confdesc->{$key}->{format};
1243     if (!$desc) {
1244         warn "invalid drive key: $key\n";
1245         return undef;
1246     }
1247     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1248     return undef if !$res;
1249     $res->{interface} = $interface;
1250     $res->{index} = $index;
1251
1252     my $error = 0;
1253     foreach my $opt (qw(bps bps_rd bps_wr)) {
1254         if (my $bps = defined(delete $res->{$opt})) {
1255             if (defined($res->{"m$opt"})) {
1256                 warn "both $opt and m$opt specified\n";
1257                 ++$error;
1258                 next;
1259             }
1260             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1261         }
1262     }
1263     return undef if $error;
1264
1265     return undef if $res->{mbps_rd} && $res->{mbps};
1266     return undef if $res->{mbps_wr} && $res->{mbps};
1267     return undef if $res->{iops_rd} && $res->{iops};
1268     return undef if $res->{iops_wr} && $res->{iops};
1269
1270     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1271         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1272         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1273         return undef if $res->{interface} eq 'virtio';
1274     }
1275
1276     if (my $size = $res->{size}) {
1277         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1278     }
1279
1280     return $res;
1281 }
1282
1283 sub print_drive {
1284     my ($vmid, $drive) = @_;
1285     my $data = { %$drive };
1286     delete $data->{$_} for qw(index interface);
1287     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1288 }
1289
1290 sub scsi_inquiry {
1291     my($fh, $noerr) = @_;
1292
1293     my $SG_IO = 0x2285;
1294     my $SG_GET_VERSION_NUM = 0x2282;
1295
1296     my $versionbuf = "\x00" x 8;
1297     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1298     if (!$ret) {
1299         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1300         return undef;
1301     }
1302     my $version = unpack("I", $versionbuf);
1303     if ($version < 30000) {
1304         die "scsi generic interface too old\n"  if !$noerr;
1305         return undef;
1306     }
1307
1308     my $buf = "\x00" x 36;
1309     my $sensebuf = "\x00" x 8;
1310     my $cmd = pack("C x3 C x1", 0x12, 36);
1311
1312     # see /usr/include/scsi/sg.h
1313     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1314
1315     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1316                       length($sensebuf), 0, length($buf), $buf,
1317                       $cmd, $sensebuf, 6000);
1318
1319     $ret = ioctl($fh, $SG_IO, $packet);
1320     if (!$ret) {
1321         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1322         return undef;
1323     }
1324
1325     my @res = unpack($sg_io_hdr_t, $packet);
1326     if ($res[17] || $res[18]) {
1327         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1328         return undef;
1329     }
1330
1331     my $res = {};
1332     (my $byte0, my $byte1, $res->{vendor},
1333      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1334
1335     $res->{removable} = $byte1 & 128 ? 1 : 0;
1336     $res->{type} = $byte0 & 31;
1337
1338     return $res;
1339 }
1340
1341 sub path_is_scsi {
1342     my ($path) = @_;
1343
1344     my $fh = IO::File->new("+<$path") || return undef;
1345     my $res = scsi_inquiry($fh, 1);
1346     close($fh);
1347
1348     return $res;
1349 }
1350
1351 sub machine_type_is_q35 {
1352     my ($conf) = @_;
1353
1354     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1355 }
1356
1357 sub print_tabletdevice_full {
1358     my ($conf) = @_;
1359
1360     my $q35 = machine_type_is_q35($conf);
1361
1362     # we use uhci for old VMs because tablet driver was buggy in older qemu
1363     my $usbbus = $q35 ? "ehci" : "uhci";
1364
1365     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1366 }
1367
1368 sub print_drivedevice_full {
1369     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1370
1371     my $device = '';
1372     my $maxdev = 0;
1373
1374     if ($drive->{interface} eq 'virtio') {
1375         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1376         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1377         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1378     } elsif ($drive->{interface} eq 'scsi') {
1379
1380         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1381         my $unit = $drive->{index} % $maxdev;
1382         my $devicetype = 'hd';
1383         my $path = '';
1384         if (drive_is_cdrom($drive)) {
1385             $devicetype = 'cd';
1386         } else {
1387             if ($drive->{file} =~ m|^/|) {
1388                 $path = $drive->{file};
1389                 if (my $info = path_is_scsi($path)) {
1390                     if ($info->{type} == 0) {
1391                         $devicetype = 'block';
1392                     } elsif ($info->{type} == 1) { # tape
1393                         $devicetype = 'generic';
1394                     }
1395                 }
1396             } else {
1397                  $path = PVE::Storage::path($storecfg, $drive->{file});
1398             }
1399
1400             if($path =~ m/^iscsi\:\/\//){
1401                 $devicetype = 'generic';
1402             }
1403         }
1404
1405         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1406             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1407         } else {
1408             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1409         }
1410
1411     } elsif ($drive->{interface} eq 'ide'){
1412         $maxdev = 2;
1413         my $controller = int($drive->{index} / $maxdev);
1414         my $unit = $drive->{index} % $maxdev;
1415         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1416
1417         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1418         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1419             $model = URI::Escape::uri_unescape($model);
1420             $device .= ",model=$model";
1421         }
1422     } elsif ($drive->{interface} eq 'sata'){
1423         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1424         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1425         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1426     } elsif ($drive->{interface} eq 'usb') {
1427         die "implement me";
1428         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1429     } else {
1430         die "unsupported interface type";
1431     }
1432
1433     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1434
1435     return $device;
1436 }
1437
1438 sub get_initiator_name {
1439     my $initiator;
1440
1441     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1442     while (defined(my $line = <$fh>)) {
1443         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1444         $initiator = $1;
1445         last;
1446     }
1447     $fh->close();
1448
1449     return $initiator;
1450 }
1451
1452 sub print_drive_full {
1453     my ($storecfg, $vmid, $drive) = @_;
1454
1455     my $path;
1456     my $volid = $drive->{file};
1457     my $format;
1458
1459     if (drive_is_cdrom($drive)) {
1460         $path = get_iso_path($storecfg, $vmid, $volid);
1461     } else {
1462         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1463         if ($storeid) {
1464             $path = PVE::Storage::path($storecfg, $volid);
1465             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1466             $format = qemu_img_format($scfg, $volname);
1467         } else {
1468             $path = $volid;
1469             $format = "raw";
1470         }
1471    }
1472
1473     my $opts = '';
1474     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1475     foreach my $o (@qemu_drive_options) {
1476         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1477     }
1478     if (my $serial = $drive->{serial}) {
1479         $serial = URI::Escape::uri_unescape($serial);
1480         $opts .= ",serial=$serial";
1481     }
1482
1483     $opts .= ",format=$format" if $format && !$drive->{format};
1484
1485     foreach my $o (qw(bps bps_rd bps_wr)) {
1486         my $v = $drive->{"m$o"};
1487         $opts .= ",$o=" . int($v*1024*1024) if $v;
1488     }
1489
1490     my $cache_direct = 0;
1491
1492     if (my $cache = $drive->{cache}) {
1493         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1494     } elsif (!drive_is_cdrom($drive)) {
1495         $opts .= ",cache=none";
1496         $cache_direct = 1;
1497     }
1498
1499     # aio native works only with O_DIRECT
1500     if (!$drive->{aio}) {
1501         if($cache_direct) {
1502             $opts .= ",aio=native";
1503         } else {
1504             $opts .= ",aio=threads";
1505         }
1506     }
1507
1508     if (!drive_is_cdrom($drive)) {
1509         my $detectzeroes;
1510         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1511             $detectzeroes = 'off';
1512         } elsif ($drive->{discard}) {
1513             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1514         } else {
1515             # This used to be our default with discard not being specified:
1516             $detectzeroes = 'on';
1517         }
1518         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1519     }
1520
1521     my $pathinfo = $path ? "file=$path," : '';
1522
1523     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1524 }
1525
1526 sub print_netdevice_full {
1527     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1528
1529     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1530
1531     my $device = $net->{model};
1532     if ($net->{model} eq 'virtio') {
1533          $device = 'virtio-net-pci';
1534      };
1535
1536     my $pciaddr = print_pci_addr("$netid", $bridges);
1537     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1538     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1539         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1540         my $vectors = $net->{queues} * 2 + 2;
1541         $tmpstr .= ",vectors=$vectors,mq=on";
1542     }
1543     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1544
1545     if ($use_old_bios_files) {
1546         my $romfile;
1547         if ($device eq 'virtio-net-pci') {
1548             $romfile = 'pxe-virtio.rom';
1549         } elsif ($device eq 'e1000') {
1550             $romfile = 'pxe-e1000.rom';
1551         } elsif ($device eq 'ne2k') {
1552             $romfile = 'pxe-ne2k_pci.rom';
1553         } elsif ($device eq 'pcnet') {
1554             $romfile = 'pxe-pcnet.rom';
1555         } elsif ($device eq 'rtl8139') {
1556             $romfile = 'pxe-rtl8139.rom';
1557         }
1558         $tmpstr .= ",romfile=$romfile" if $romfile;
1559     }
1560
1561     return $tmpstr;
1562 }
1563
1564 sub print_netdev_full {
1565     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1566
1567     my $i = '';
1568     if ($netid =~ m/^net(\d+)$/) {
1569         $i = int($1);
1570     }
1571
1572     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1573
1574     my $ifname = "tap${vmid}i$i";
1575
1576     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1577     die "interface name '$ifname' is too long (max 15 character)\n"
1578         if length($ifname) >= 16;
1579
1580     my $vhostparam = '';
1581     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1582
1583     my $vmname = $conf->{name} || "vm$vmid";
1584
1585     my $netdev = "";
1586     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1587
1588     if ($net->{bridge}) {
1589         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1590     } else {
1591         $netdev = "type=user,id=$netid,hostname=$vmname";
1592     }
1593
1594     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1595
1596     return $netdev;
1597 }
1598
1599 sub drive_is_cdrom {
1600     my ($drive) = @_;
1601
1602     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1603
1604 }
1605
1606 sub parse_number_sets {
1607     my ($set) = @_;
1608     my $res = [];
1609     foreach my $part (split(/;/, $set)) {
1610         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1611             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1612             push @$res, [ $1, $2 ];
1613         } else {
1614             die "invalid range: $part\n";
1615         }
1616     }
1617     return $res;
1618 }
1619
1620 sub parse_numa {
1621     my ($data) = @_;
1622
1623     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1624     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1625     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1626     return $res;
1627 }
1628
1629 sub parse_hostpci {
1630     my ($value) = @_;
1631
1632     return undef if !$value;
1633
1634     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1635
1636     my @idlist = split(/;/, $res->{host});
1637     delete $res->{host};
1638     foreach my $id (@idlist) {
1639         if ($id =~ /^$PCIRE$/) {
1640             push @{$res->{pciid}}, { id => $1, function => ($2//'0') };
1641         } else {
1642             # should have been caught by parse_property_string already
1643             die "failed to parse PCI id: $id\n";
1644         }
1645     }
1646     return $res;
1647 }
1648
1649 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1650 sub parse_net {
1651     my ($data) = @_;
1652
1653     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1654     if ($@) {
1655         warn $@;
1656         return undef;
1657     }
1658     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1659     return $res;
1660 }
1661
1662 sub print_net {
1663     my $net = shift;
1664
1665     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1666 }
1667
1668 sub add_random_macs {
1669     my ($settings) = @_;
1670
1671     foreach my $opt (keys %$settings) {
1672         next if $opt !~ m/^net(\d+)$/;
1673         my $net = parse_net($settings->{$opt});
1674         next if !$net;
1675         $settings->{$opt} = print_net($net);
1676     }
1677 }
1678
1679 sub vm_is_volid_owner {
1680     my ($storecfg, $vmid, $volid) = @_;
1681
1682     if ($volid !~  m|^/|) {
1683         my ($path, $owner);
1684         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1685         if ($owner && ($owner == $vmid)) {
1686             return 1;
1687         }
1688     }
1689
1690     return undef;
1691 }
1692
1693 sub split_flagged_list {
1694     my $text = shift || '';
1695     $text =~ s/[,;]/ /g;
1696     $text =~ s/^\s+//;
1697     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1698 }
1699
1700 sub join_flagged_list {
1701     my ($how, $lst) = @_;
1702     join $how, map { $lst->{$_} . $_ } keys %$lst;
1703 }
1704
1705 sub vmconfig_delete_pending_option {
1706     my ($conf, $key, $force) = @_;
1707
1708     delete $conf->{pending}->{$key};
1709     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1710     $pending_delete_hash->{$key} = $force ? '!' : '';
1711     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1712 }
1713
1714 sub vmconfig_undelete_pending_option {
1715     my ($conf, $key) = @_;
1716
1717     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1718     delete $pending_delete_hash->{$key};
1719
1720     if (%$pending_delete_hash) {
1721         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1722     } else {
1723         delete $conf->{pending}->{delete};
1724     }
1725 }
1726
1727 sub vmconfig_register_unused_drive {
1728     my ($storecfg, $vmid, $conf, $drive) = @_;
1729
1730     if (!drive_is_cdrom($drive)) {
1731         my $volid = $drive->{file};
1732         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1733             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1734         }
1735     }
1736 }
1737
1738 sub vmconfig_cleanup_pending {
1739     my ($conf) = @_;
1740
1741     # remove pending changes when nothing changed
1742     my $changes;
1743     foreach my $opt (keys %{$conf->{pending}}) {
1744         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1745             $changes = 1;
1746             delete $conf->{pending}->{$opt};
1747         }
1748     }
1749
1750     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1751     my $pending_delete_hash = {};
1752     while (my ($opt, $force) = each %$current_delete_hash) {
1753         if (defined($conf->{$opt})) {
1754             $pending_delete_hash->{$opt} = $force;
1755         } else {
1756             $changes = 1;
1757         }
1758     }
1759
1760     if (%$pending_delete_hash) {
1761         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1762     } else {
1763         delete $conf->{pending}->{delete};
1764     }
1765
1766     return $changes;
1767 }
1768
1769 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1770 my $smbios1_fmt = {
1771     uuid => {
1772         type => 'string',
1773         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1774         format_description => 'UUID',
1775         optional => 1,
1776     },
1777     version => {
1778         type => 'string',
1779         pattern => '\S+',
1780         format_description => 'str',
1781         optional => 1,
1782     },
1783     serial => {
1784         type => 'string',
1785         pattern => '\S+',
1786         format_description => 'str',
1787         optional => 1,
1788     },
1789     manufacturer => {
1790         type => 'string',
1791         pattern => '\S+',
1792         format_description => 'name',
1793         optional => 1,
1794     },
1795     product => {
1796         type => 'string',
1797         pattern => '\S+',
1798         format_description => 'name',
1799         optional => 1,
1800     },
1801     sku => {
1802         type => 'string',
1803         pattern => '\S+',
1804         format_description => 'str',
1805         optional => 1,
1806     },
1807     family => {
1808         type => 'string',
1809         pattern => '\S+',
1810         format_description => 'str',
1811         optional => 1,
1812     },
1813 };
1814
1815 sub parse_smbios1 {
1816     my ($data) = @_;
1817
1818     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1819     warn $@ if $@;
1820     return $res;
1821 }
1822
1823 sub print_smbios1 {
1824     my ($smbios1) = @_;
1825     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1826 }
1827
1828 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1829
1830 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1831 sub verify_bootdisk {
1832     my ($value, $noerr) = @_;
1833
1834     return $value if is_valid_drivename($value);
1835
1836     return undef if $noerr;
1837
1838     die "invalid boot disk '$value'\n";
1839 }
1840
1841 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1842 sub verify_net {
1843     my ($value, $noerr) = @_;
1844
1845     return $value if parse_net($value);
1846
1847     return undef if $noerr;
1848
1849     die "unable to parse network options\n";
1850 }
1851
1852 sub parse_watchdog {
1853     my ($value) = @_;
1854
1855     return undef if !$value;
1856
1857     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1858     warn $@ if $@;
1859     return $res;
1860 }
1861
1862 sub parse_usb_device {
1863     my ($value) = @_;
1864
1865     return undef if !$value;
1866
1867     my $res = {};
1868     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1869         $res->{vendorid} = $2;
1870         $res->{productid} = $4;
1871     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1872         $res->{hostbus} = $1;
1873         $res->{hostport} = $2;
1874     } elsif ($value =~ m/^spice$/i) {
1875         $res->{spice} = 1;
1876     } else {
1877         return undef;
1878     }
1879
1880     return $res;
1881 }
1882
1883 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1884 sub verify_usb_device {
1885     my ($value, $noerr) = @_;
1886
1887     return $value if parse_usb_device($value);
1888
1889     return undef if $noerr;
1890
1891     die "unable to parse usb device\n";
1892 }
1893
1894 # add JSON properties for create and set function
1895 sub json_config_properties {
1896     my $prop = shift;
1897
1898     foreach my $opt (keys %$confdesc) {
1899         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1900         $prop->{$opt} = $confdesc->{$opt};
1901     }
1902
1903     return $prop;
1904 }
1905
1906 sub check_type {
1907     my ($key, $value) = @_;
1908
1909     die "unknown setting '$key'\n" if !$confdesc->{$key};
1910
1911     my $type = $confdesc->{$key}->{type};
1912
1913     if (!defined($value)) {
1914         die "got undefined value\n";
1915     }
1916
1917     if ($value =~ m/[\n\r]/) {
1918         die "property contains a line feed\n";
1919     }
1920
1921     if ($type eq 'boolean') {
1922         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1923         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1924         die "type check ('boolean') failed - got '$value'\n";
1925     } elsif ($type eq 'integer') {
1926         return int($1) if $value =~ m/^(\d+)$/;
1927         die "type check ('integer') failed - got '$value'\n";
1928     } elsif ($type eq 'number') {
1929         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1930         die "type check ('number') failed - got '$value'\n";
1931     } elsif ($type eq 'string') {
1932         if (my $fmt = $confdesc->{$key}->{format}) {
1933             if ($fmt eq 'pve-qm-drive') {
1934                 # special case - we need to pass $key to parse_drive()
1935                 my $drive = parse_drive($key, $value);
1936                 return $value if $drive;
1937                 die "unable to parse drive options\n";
1938             }
1939             PVE::JSONSchema::check_format($fmt, $value);
1940             return $value;
1941         }
1942         $value =~ s/^\"(.*)\"$/$1/;
1943         return $value;
1944     } else {
1945         die "internal error"
1946     }
1947 }
1948
1949 sub check_iommu_support{
1950     #fixme : need to check IOMMU support
1951     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1952
1953     my $iommu=1;
1954     return $iommu;
1955
1956 }
1957
1958 sub touch_config {
1959     my ($vmid) = @_;
1960
1961     my $conf = PVE::QemuConfig->config_file($vmid);
1962     utime undef, undef, $conf;
1963 }
1964
1965 sub destroy_vm {
1966     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1967
1968     my $conffile = PVE::QemuConfig->config_file($vmid);
1969
1970     my $conf = PVE::QemuConfig->load_config($vmid);
1971
1972     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1973
1974     # only remove disks owned by this VM
1975     foreach_drive($conf, sub {
1976         my ($ds, $drive) = @_;
1977
1978         return if drive_is_cdrom($drive);
1979
1980         my $volid = $drive->{file};
1981
1982         return if !$volid || $volid =~ m|^/|;
1983
1984         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1985         return if !$path || !$owner || ($owner != $vmid);
1986
1987         PVE::Storage::vdisk_free($storecfg, $volid);
1988     });
1989
1990     if ($keep_empty_config) {
1991         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1992     } else {
1993         unlink $conffile;
1994     }
1995
1996     # also remove unused disk
1997     eval {
1998         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1999
2000         eval {
2001             PVE::Storage::foreach_volid($dl, sub {
2002                 my ($volid, $sid, $volname, $d) = @_;
2003                 PVE::Storage::vdisk_free($storecfg, $volid);
2004             });
2005         };
2006         warn $@ if $@;
2007
2008     };
2009     warn $@ if $@;
2010 }
2011
2012 sub parse_vm_config {
2013     my ($filename, $raw) = @_;
2014
2015     return undef if !defined($raw);
2016
2017     my $res = {
2018         digest => Digest::SHA::sha1_hex($raw),
2019         snapshots => {},
2020         pending => {},
2021     };
2022
2023     $filename =~ m|/qemu-server/(\d+)\.conf$|
2024         || die "got strange filename '$filename'";
2025
2026     my $vmid = $1;
2027
2028     my $conf = $res;
2029     my $descr;
2030     my $section = '';
2031
2032     my @lines = split(/\n/, $raw);
2033     foreach my $line (@lines) {
2034         next if $line =~ m/^\s*$/;
2035
2036         if ($line =~ m/^\[PENDING\]\s*$/i) {
2037             $section = 'pending';
2038             if (defined($descr)) {
2039                 $descr =~ s/\s+$//;
2040                 $conf->{description} = $descr;
2041             }
2042             $descr = undef;
2043             $conf = $res->{$section} = {};
2044             next;
2045
2046         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2047             $section = $1;
2048             if (defined($descr)) {
2049                 $descr =~ s/\s+$//;
2050                 $conf->{description} = $descr;
2051             }
2052             $descr = undef;
2053             $conf = $res->{snapshots}->{$section} = {};
2054             next;
2055         }
2056
2057         if ($line =~ m/^\#(.*)\s*$/) {
2058             $descr = '' if !defined($descr);
2059             $descr .= PVE::Tools::decode_text($1) . "\n";
2060             next;
2061         }
2062
2063         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2064             $descr = '' if !defined($descr);
2065             $descr .= PVE::Tools::decode_text($2);
2066         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2067             $conf->{snapstate} = $1;
2068         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2069             my $key = $1;
2070             my $value = $2;
2071             $conf->{$key} = $value;
2072         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2073             my $value = $1;
2074             if ($section eq 'pending') {
2075                 $conf->{delete} = $value; # we parse this later
2076             } else {
2077                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2078             }
2079         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2080             my $key = $1;
2081             my $value = $2;
2082             eval { $value = check_type($key, $value); };
2083             if ($@) {
2084                 warn "vm $vmid - unable to parse value of '$key' - $@";
2085             } else {
2086                 my $fmt = $confdesc->{$key}->{format};
2087                 if ($fmt && $fmt eq 'pve-qm-drive') {
2088                     my $v = parse_drive($key, $value);
2089                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2090                         $v->{file} = $volid;
2091                         $value = print_drive($vmid, $v);
2092                     } else {
2093                         warn "vm $vmid - unable to parse value of '$key'\n";
2094                         next;
2095                     }
2096                 }
2097
2098                 if ($key eq 'cdrom') {
2099                     $conf->{ide2} = $value;
2100                 } else {
2101                     $conf->{$key} = $value;
2102                 }
2103             }
2104         }
2105     }
2106
2107     if (defined($descr)) {
2108         $descr =~ s/\s+$//;
2109         $conf->{description} = $descr;
2110     }
2111     delete $res->{snapstate}; # just to be sure
2112
2113     return $res;
2114 }
2115
2116 sub write_vm_config {
2117     my ($filename, $conf) = @_;
2118
2119     delete $conf->{snapstate}; # just to be sure
2120
2121     if ($conf->{cdrom}) {
2122         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2123         $conf->{ide2} = $conf->{cdrom};
2124         delete $conf->{cdrom};
2125     }
2126
2127     # we do not use 'smp' any longer
2128     if ($conf->{sockets}) {
2129         delete $conf->{smp};
2130     } elsif ($conf->{smp}) {
2131         $conf->{sockets} = $conf->{smp};
2132         delete $conf->{cores};
2133         delete $conf->{smp};
2134     }
2135
2136     my $used_volids = {};
2137
2138     my $cleanup_config = sub {
2139         my ($cref, $pending, $snapname) = @_;
2140
2141         foreach my $key (keys %$cref) {
2142             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2143                 $key eq 'snapstate' || $key eq 'pending';
2144             my $value = $cref->{$key};
2145             if ($key eq 'delete') {
2146                 die "propertry 'delete' is only allowed in [PENDING]\n"
2147                     if !$pending;
2148                 # fixme: check syntax?
2149                 next;
2150             }
2151             eval { $value = check_type($key, $value); };
2152             die "unable to parse value of '$key' - $@" if $@;
2153
2154             $cref->{$key} = $value;
2155
2156             if (!$snapname && is_valid_drivename($key)) {
2157                 my $drive = parse_drive($key, $value);
2158                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2159             }
2160         }
2161     };
2162
2163     &$cleanup_config($conf);
2164
2165     &$cleanup_config($conf->{pending}, 1);
2166
2167     foreach my $snapname (keys %{$conf->{snapshots}}) {
2168         die "internal error" if $snapname eq 'pending';
2169         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2170     }
2171
2172     # remove 'unusedX' settings if we re-add a volume
2173     foreach my $key (keys %$conf) {
2174         my $value = $conf->{$key};
2175         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2176             delete $conf->{$key};
2177         }
2178     }
2179
2180     my $generate_raw_config = sub {
2181         my ($conf, $pending) = @_;
2182
2183         my $raw = '';
2184
2185         # add description as comment to top of file
2186         if (defined(my $descr = $conf->{description})) {
2187             if ($descr) {
2188                 foreach my $cl (split(/\n/, $descr)) {
2189                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2190                 }
2191             } else {
2192                 $raw .= "#\n" if $pending;
2193             }
2194         }
2195
2196         foreach my $key (sort keys %$conf) {
2197             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2198             $raw .= "$key: $conf->{$key}\n";
2199         }
2200         return $raw;
2201     };
2202
2203     my $raw = &$generate_raw_config($conf);
2204
2205     if (scalar(keys %{$conf->{pending}})){
2206         $raw .= "\n[PENDING]\n";
2207         $raw .= &$generate_raw_config($conf->{pending}, 1);
2208     }
2209
2210     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2211         $raw .= "\n[$snapname]\n";
2212         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2213     }
2214
2215     return $raw;
2216 }
2217
2218 sub load_defaults {
2219
2220     my $res = {};
2221
2222     # we use static defaults from our JSON schema configuration
2223     foreach my $key (keys %$confdesc) {
2224         if (defined(my $default = $confdesc->{$key}->{default})) {
2225             $res->{$key} = $default;
2226         }
2227     }
2228
2229     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2230     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2231
2232     return $res;
2233 }
2234
2235 sub config_list {
2236     my $vmlist = PVE::Cluster::get_vmlist();
2237     my $res = {};
2238     return $res if !$vmlist || !$vmlist->{ids};
2239     my $ids = $vmlist->{ids};
2240
2241     foreach my $vmid (keys %$ids) {
2242         my $d = $ids->{$vmid};
2243         next if !$d->{node} || $d->{node} ne $nodename;
2244         next if !$d->{type} || $d->{type} ne 'qemu';
2245         $res->{$vmid}->{exists} = 1;
2246     }
2247     return $res;
2248 }
2249
2250 # test if VM uses local resources (to prevent migration)
2251 sub check_local_resources {
2252     my ($conf, $noerr) = @_;
2253
2254     my $loc_res = 0;
2255
2256     $loc_res = 1 if $conf->{hostusb}; # old syntax
2257     $loc_res = 1 if $conf->{hostpci}; # old syntax
2258
2259     foreach my $k (keys %$conf) {
2260         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2261         # sockets are safe: they will recreated be on the target side post-migrate
2262         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2263         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2264     }
2265
2266     die "VM uses local resources\n" if $loc_res && !$noerr;
2267
2268     return $loc_res;
2269 }
2270
2271 # check if used storages are available on all nodes (use by migrate)
2272 sub check_storage_availability {
2273     my ($storecfg, $conf, $node) = @_;
2274
2275     foreach_drive($conf, sub {
2276         my ($ds, $drive) = @_;
2277
2278         my $volid = $drive->{file};
2279         return if !$volid;
2280
2281         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2282         return if !$sid;
2283
2284         # check if storage is available on both nodes
2285         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2286         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2287    });
2288 }
2289
2290 # list nodes where all VM images are available (used by has_feature API)
2291 sub shared_nodes {
2292     my ($conf, $storecfg) = @_;
2293
2294     my $nodelist = PVE::Cluster::get_nodelist();
2295     my $nodehash = { map { $_ => 1 } @$nodelist };
2296     my $nodename = PVE::INotify::nodename();
2297
2298     foreach_drive($conf, sub {
2299         my ($ds, $drive) = @_;
2300
2301         my $volid = $drive->{file};
2302         return if !$volid;
2303
2304         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2305         if ($storeid) {
2306             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2307             if ($scfg->{disable}) {
2308                 $nodehash = {};
2309             } elsif (my $avail = $scfg->{nodes}) {
2310                 foreach my $node (keys %$nodehash) {
2311                     delete $nodehash->{$node} if !$avail->{$node};
2312                 }
2313             } elsif (!$scfg->{shared}) {
2314                 foreach my $node (keys %$nodehash) {
2315                     delete $nodehash->{$node} if $node ne $nodename
2316                 }
2317             }
2318         }
2319     });
2320
2321     return $nodehash
2322 }
2323
2324 sub check_cmdline {
2325     my ($pidfile, $pid) = @_;
2326
2327     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2328     if (defined($fh)) {
2329         my $line = <$fh>;
2330         $fh->close;
2331         return undef if !$line;
2332         my @param = split(/\0/, $line);
2333
2334         my $cmd = $param[0];
2335         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2336
2337         for (my $i = 0; $i < scalar (@param); $i++) {
2338             my $p = $param[$i];
2339             next if !$p;
2340             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2341                 my $p = $param[$i+1];
2342                 return 1 if $p && ($p eq $pidfile);
2343                 return undef;
2344             }
2345         }
2346     }
2347     return undef;
2348 }
2349
2350 sub check_running {
2351     my ($vmid, $nocheck, $node) = @_;
2352
2353     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2354
2355     die "unable to find configuration file for VM $vmid - no such machine\n"
2356         if !$nocheck && ! -f $filename;
2357
2358     my $pidfile = pidfile_name($vmid);
2359
2360     if (my $fd = IO::File->new("<$pidfile")) {
2361         my $st = stat($fd);
2362         my $line = <$fd>;
2363         close($fd);
2364
2365         my $mtime = $st->mtime;
2366         if ($mtime > time()) {
2367             warn "file '$filename' modified in future\n";
2368         }
2369
2370         if ($line =~ m/^(\d+)$/) {
2371             my $pid = $1;
2372             if (check_cmdline($pidfile, $pid)) {
2373                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2374                     return $pid;
2375                 }
2376             }
2377         }
2378     }
2379
2380     return undef;
2381 }
2382
2383 sub vzlist {
2384
2385     my $vzlist = config_list();
2386
2387     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2388
2389     while (defined(my $de = $fd->read)) {
2390         next if $de !~ m/^(\d+)\.pid$/;
2391         my $vmid = $1;
2392         next if !defined($vzlist->{$vmid});
2393         if (my $pid = check_running($vmid)) {
2394             $vzlist->{$vmid}->{pid} = $pid;
2395         }
2396     }
2397
2398     return $vzlist;
2399 }
2400
2401 sub disksize {
2402     my ($storecfg, $conf) = @_;
2403
2404     my $bootdisk = $conf->{bootdisk};
2405     return undef if !$bootdisk;
2406     return undef if !is_valid_drivename($bootdisk);
2407
2408     return undef if !$conf->{$bootdisk};
2409
2410     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2411     return undef if !defined($drive);
2412
2413     return undef if drive_is_cdrom($drive);
2414
2415     my $volid = $drive->{file};
2416     return undef if !$volid;
2417
2418     return $drive->{size};
2419 }
2420
2421 my $last_proc_pid_stat;
2422
2423 # get VM status information
2424 # This must be fast and should not block ($full == false)
2425 # We only query KVM using QMP if $full == true (this can be slow)
2426 sub vmstatus {
2427     my ($opt_vmid, $full) = @_;
2428
2429     my $res = {};
2430
2431     my $storecfg = PVE::Storage::config();
2432
2433     my $list = vzlist();
2434     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2435
2436     my $cpucount = $cpuinfo->{cpus} || 1;
2437
2438     foreach my $vmid (keys %$list) {
2439         next if $opt_vmid && ($vmid ne $opt_vmid);
2440
2441         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2442         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2443
2444         my $d = {};
2445         $d->{pid} = $list->{$vmid}->{pid};
2446
2447         # fixme: better status?
2448         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2449
2450         my $size = disksize($storecfg, $conf);
2451         if (defined($size)) {
2452             $d->{disk} = 0; # no info available
2453             $d->{maxdisk} = $size;
2454         } else {
2455             $d->{disk} = 0;
2456             $d->{maxdisk} = 0;
2457         }
2458
2459         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2460         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2461         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2462
2463         $d->{name} = $conf->{name} || "VM $vmid";
2464         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2465
2466         if ($conf->{balloon}) {
2467             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2468             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2469         }
2470
2471         $d->{uptime} = 0;
2472         $d->{cpu} = 0;
2473         $d->{mem} = 0;
2474
2475         $d->{netout} = 0;
2476         $d->{netin} = 0;
2477
2478         $d->{diskread} = 0;
2479         $d->{diskwrite} = 0;
2480
2481         $d->{template} = PVE::QemuConfig->is_template($conf);
2482
2483         $res->{$vmid} = $d;
2484     }
2485
2486     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2487     foreach my $dev (keys %$netdev) {
2488         next if $dev !~ m/^tap([1-9]\d*)i/;
2489         my $vmid = $1;
2490         my $d = $res->{$vmid};
2491         next if !$d;
2492
2493         $d->{netout} += $netdev->{$dev}->{receive};
2494         $d->{netin} += $netdev->{$dev}->{transmit};
2495
2496         if ($full) {
2497             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2498             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2499         }
2500
2501     }
2502
2503     my $ctime = gettimeofday;
2504
2505     foreach my $vmid (keys %$list) {
2506
2507         my $d = $res->{$vmid};
2508         my $pid = $d->{pid};
2509         next if !$pid;
2510
2511         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2512         next if !$pstat; # not running
2513
2514         my $used = $pstat->{utime} + $pstat->{stime};
2515
2516         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2517
2518         if ($pstat->{vsize}) {
2519             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2520         }
2521
2522         my $old = $last_proc_pid_stat->{$pid};
2523         if (!$old) {
2524             $last_proc_pid_stat->{$pid} = {
2525                 time => $ctime,
2526                 used => $used,
2527                 cpu => 0,
2528             };
2529             next;
2530         }
2531
2532         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2533
2534         if ($dtime > 1000) {
2535             my $dutime = $used -  $old->{used};
2536
2537             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2538             $last_proc_pid_stat->{$pid} = {
2539                 time => $ctime,
2540                 used => $used,
2541                 cpu => $d->{cpu},
2542             };
2543         } else {
2544             $d->{cpu} = $old->{cpu};
2545         }
2546     }
2547
2548     return $res if !$full;
2549
2550     my $qmpclient = PVE::QMPClient->new();
2551
2552     my $ballooncb = sub {
2553         my ($vmid, $resp) = @_;
2554
2555         my $info = $resp->{'return'};
2556         return if !$info->{max_mem};
2557
2558         my $d = $res->{$vmid};
2559
2560         # use memory assigned to VM
2561         $d->{maxmem} = $info->{max_mem};
2562         $d->{balloon} = $info->{actual};
2563
2564         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2565             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2566             $d->{freemem} = $info->{free_mem};
2567         }
2568
2569         $d->{ballooninfo} = $info;
2570     };
2571
2572     my $blockstatscb = sub {
2573         my ($vmid, $resp) = @_;
2574         my $data = $resp->{'return'} || [];
2575         my $totalrdbytes = 0;
2576         my $totalwrbytes = 0;
2577
2578         for my $blockstat (@$data) {
2579             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2580             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2581
2582             $blockstat->{device} =~ s/drive-//;
2583             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2584         }
2585         $res->{$vmid}->{diskread} = $totalrdbytes;
2586         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2587     };
2588
2589     my $statuscb = sub {
2590         my ($vmid, $resp) = @_;
2591
2592         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2593         # this fails if ballon driver is not loaded, so this must be
2594         # the last commnand (following command are aborted if this fails).
2595         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2596
2597         my $status = 'unknown';
2598         if (!defined($status = $resp->{'return'}->{status})) {
2599             warn "unable to get VM status\n";
2600             return;
2601         }
2602
2603         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2604     };
2605
2606     foreach my $vmid (keys %$list) {
2607         next if $opt_vmid && ($vmid ne $opt_vmid);
2608         next if !$res->{$vmid}->{pid}; # not running
2609         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2610     }
2611
2612     $qmpclient->queue_execute(undef, 1);
2613
2614     foreach my $vmid (keys %$list) {
2615         next if $opt_vmid && ($vmid ne $opt_vmid);
2616         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2617     }
2618
2619     return $res;
2620 }
2621
2622 sub foreach_dimm {
2623     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2624
2625     my $dimm_id = 0;
2626     my $current_size = 1024;
2627     my $dimm_size = 512;
2628     return if $current_size == $memory;
2629
2630     for (my $j = 0; $j < 8; $j++) {
2631         for (my $i = 0; $i < 32; $i++) {
2632             my $name = "dimm${dimm_id}";
2633             $dimm_id++;
2634             my $numanode = $i % $sockets;
2635             $current_size += $dimm_size;
2636             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2637             return  $current_size if $current_size >= $memory;
2638         }
2639         $dimm_size *= 2;
2640     }
2641 }
2642
2643 sub foreach_reverse_dimm {
2644     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2645
2646     my $dimm_id = 253;
2647     my $current_size = 4177920;
2648     my $dimm_size = 65536;
2649     return if $current_size == $memory;
2650
2651     for (my $j = 0; $j < 8; $j++) {
2652         for (my $i = 0; $i < 32; $i++) {
2653             my $name = "dimm${dimm_id}";
2654             $dimm_id--;
2655             my $numanode = $i % $sockets;
2656             $current_size -= $dimm_size;
2657             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2658             return  $current_size if $current_size <= $memory;
2659         }
2660         $dimm_size /= 2;
2661     }
2662 }
2663
2664 sub foreach_drive {
2665     my ($conf, $func) = @_;
2666
2667     foreach my $ds (valid_drive_names()) {
2668         next if !defined($conf->{$ds});
2669
2670         my $drive = parse_drive($ds, $conf->{$ds});
2671         next if !$drive;
2672
2673         &$func($ds, $drive);
2674     }
2675 }
2676
2677 sub foreach_volid {
2678     my ($conf, $func) = @_;
2679
2680     my $volhash = {};
2681
2682     my $test_volid = sub {
2683         my ($volid, $is_cdrom) = @_;
2684
2685         return if !$volid;
2686
2687         $volhash->{$volid} = $is_cdrom || 0;
2688     };
2689
2690     foreach_drive($conf, sub {
2691         my ($ds, $drive) = @_;
2692         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2693     });
2694
2695     foreach my $snapname (keys %{$conf->{snapshots}}) {
2696         my $snap = $conf->{snapshots}->{$snapname};
2697         &$test_volid($snap->{vmstate}, 0);
2698         foreach_drive($snap, sub {
2699             my ($ds, $drive) = @_;
2700             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2701         });
2702     }
2703
2704     foreach my $volid (keys %$volhash) {
2705         &$func($volid, $volhash->{$volid});
2706     }
2707 }
2708
2709 sub vga_conf_has_spice {
2710     my ($vga) = @_;
2711
2712     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2713
2714     return $1 || 1;
2715 }
2716
2717 sub config_to_command {
2718     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2719
2720     my $cmd = [];
2721     my $globalFlags = [];
2722     my $machineFlags = [];
2723     my $rtcFlags = [];
2724     my $cpuFlags = [];
2725     my $devices = [];
2726     my $pciaddr = '';
2727     my $bridges = {};
2728     my $kvmver = kvm_user_version();
2729     my $vernum = 0; # unknown
2730     my $ostype = $conf->{ostype};
2731     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2732         $vernum = $1*1000000+$2*1000;
2733     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2734         $vernum = $1*1000000+$2*1000+$3;
2735     }
2736
2737     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2738
2739     my $have_ovz = -f '/proc/vz/vestat';
2740
2741     my $q35 = machine_type_is_q35($conf);
2742     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2743     my $machine_type = $forcemachine || $conf->{machine};
2744     my $use_old_bios_files = undef;
2745     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2746
2747     my $cpuunits = defined($conf->{cpuunits}) ?
2748             $conf->{cpuunits} : $defaults->{cpuunits};
2749
2750     push @$cmd, '/usr/bin/systemd-run';
2751     push @$cmd, '--scope';
2752     push @$cmd, '--slice', "qemu";
2753     push @$cmd, '--unit', $vmid;
2754     # set KillMode=none, so that systemd don't kill those scopes
2755     # at shutdown (pve-manager service should stop the VMs instead)
2756     push @$cmd, '-p', "KillMode=none";
2757     push @$cmd, '-p', "CPUShares=$cpuunits";
2758     if ($conf->{cpulimit}) {
2759         my $cpulimit = int($conf->{cpulimit} * 100);
2760         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2761     }
2762
2763     push @$cmd, '/usr/bin/kvm';
2764
2765     push @$cmd, '-id', $vmid;
2766
2767     my $use_virtio = 0;
2768
2769     my $qmpsocket = qmp_socket($vmid);
2770     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2771     push @$cmd, '-mon', "chardev=qmp,mode=control";
2772
2773
2774     push @$cmd, '-pidfile' , pidfile_name($vmid);
2775
2776     push @$cmd, '-daemonize';
2777
2778     if ($conf->{smbios1}) {
2779         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2780     }
2781
2782     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2783         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2784         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2785         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2786         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2787         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2788         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2789     }
2790
2791     if ($q35) {
2792         # the q35 chipset support native usb2, so we enable usb controller
2793         # by default for this machine type
2794         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2795     } else {
2796         $pciaddr = print_pci_addr("piix3", $bridges);
2797         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2798
2799         my $use_usb2 = 0;
2800         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2801             next if !$conf->{"usb$i"};
2802             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2803             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2804             $use_usb2 = 1;
2805         }
2806         # include usb device config
2807         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2808     }
2809
2810     # add usb3 controller if needed
2811
2812     my $use_usb3 = 0;
2813     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2814         next if !$conf->{"usb$i"};
2815         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2816         next if !$d || !$d->{usb3};
2817         $use_usb3 = 1;
2818     }
2819
2820     $pciaddr = print_pci_addr("xhci", $bridges);
2821     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2822
2823     my $vga = $conf->{vga};
2824
2825     my $qxlnum = vga_conf_has_spice($vga);
2826     $vga = 'qxl' if $qxlnum;
2827
2828     if (!$vga) {
2829         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2830                                 $conf->{ostype} eq 'win7' ||
2831                                 $conf->{ostype} eq 'w2k8')) {
2832             $vga = 'std';
2833         } else {
2834             $vga = 'cirrus';
2835         }
2836     }
2837
2838     # enable absolute mouse coordinates (needed by vnc)
2839     my $tablet;
2840     if (defined($conf->{tablet})) {
2841         $tablet = $conf->{tablet};
2842     } else {
2843         $tablet = $defaults->{tablet};
2844         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2845         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2846     }
2847
2848     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2849
2850     my $kvm_off = 0;
2851     # host pci devices
2852     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2853         my $d = parse_hostpci($conf->{"hostpci$i"});
2854         next if !$d;
2855
2856         my $pcie = $d->{pcie};
2857         if($pcie){
2858             die "q35 machine model is not enabled" if !$q35;
2859             $pciaddr = print_pcie_addr("hostpci$i");
2860         }else{
2861             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2862         }
2863
2864         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2865         my $xvga = '';
2866         if ($d->{'x-vga'}) {
2867             $xvga = ',x-vga=on';
2868             $kvm_off = 1;
2869             $vga = 'none';
2870             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2871                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2872             }
2873             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2874                 $xvga = "";
2875             }
2876         }
2877         my $pcidevices = $d->{pciid};
2878         my $multifunction = 1 if @$pcidevices > 1;
2879
2880         my $j=0;
2881         foreach my $pcidevice (@$pcidevices) {
2882
2883             my $id = "hostpci$i";
2884             $id .= ".$j" if $multifunction;
2885             my $addr = $pciaddr;
2886             $addr .= ".$j" if $multifunction;
2887             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2888
2889             if($j == 0){
2890                 $devicestr .= "$rombar$xvga";
2891                 $devicestr .= ",multifunction=on" if $multifunction;
2892             }
2893
2894             push @$devices, '-device', $devicestr;
2895             $j++;
2896         }
2897     }
2898
2899     # usb devices
2900     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2901         next if !$conf->{"usb$i"};
2902         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2903         next if !$d;
2904
2905         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2906         my $usbbus = '';
2907         if (defined($d->{usb3}) && $d->{usb3}) {
2908             $usbbus = ',bus=xhci.0';
2909         }
2910
2911         if (defined($d->{host})) {
2912             $d = parse_usb_device($d->{host});
2913             if (defined($d->{vendorid}) && defined($d->{productid})) {
2914                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2915             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2916                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2917             } elsif (defined($d->{spice}) && $d->{spice}) {
2918                 # usb redir support for spice, currently no usb3
2919                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2920                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2921             }
2922         }
2923     }
2924
2925     # serial devices
2926     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2927         if (my $path = $conf->{"serial$i"}) {
2928             if ($path eq 'socket') {
2929                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2930                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2931                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2932             } else {
2933                 die "no such serial device\n" if ! -c $path;
2934                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2935                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2936             }
2937         }
2938     }
2939
2940     # parallel devices
2941     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2942         if (my $path = $conf->{"parallel$i"}) {
2943             die "no such parallel device\n" if ! -c $path;
2944             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2945             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2946             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2947         }
2948     }
2949
2950     my $vmname = $conf->{name} || "vm$vmid";
2951
2952     push @$cmd, '-name', $vmname;
2953
2954     my $sockets = 1;
2955     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2956     $sockets = $conf->{sockets} if  $conf->{sockets};
2957
2958     my $cores = $conf->{cores} || 1;
2959
2960     my $maxcpus = $sockets * $cores;
2961
2962     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2963
2964     my $allowed_vcpus = $cpuinfo->{cpus};
2965
2966     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2967         if ($allowed_vcpus < $maxcpus);
2968
2969     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2970
2971     push @$cmd, '-nodefaults';
2972
2973     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2974
2975     my $bootindex_hash = {};
2976     my $i = 1;
2977     foreach my $o (split(//, $bootorder)) {
2978         $bootindex_hash->{$o} = $i*100;
2979         $i++;
2980     }
2981
2982     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2983
2984     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2985
2986     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2987
2988     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2989
2990     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2991         my $socket = vnc_socket($vmid);
2992         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2993     } else {
2994         push @$cmd, '-nographic';
2995     }
2996
2997     # time drift fix
2998     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2999
3000     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3001     my $useLocaltime = $conf->{localtime};
3002
3003     if ($ostype) {
3004         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3005
3006         if ($ostype =~ m/^w/) { # windows
3007             $useLocaltime = 1 if !defined($conf->{localtime});
3008
3009             # use time drift fix when acpi is enabled
3010             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3011                 $tdf = 1 if !defined($conf->{tdf});
3012             }
3013         }
3014
3015         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3016             $ostype eq 'wvista') {
3017             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3018             push @$cmd, '-no-hpet';
3019             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3020                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3021                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3022                 push @$cpuFlags , 'hv_time' if !$nokvm;
3023
3024             } else {
3025                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3026             }
3027         }
3028
3029         if ($ostype eq 'win7' || $ostype eq 'win8') {
3030             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3031         }
3032     }
3033
3034     push @$rtcFlags, 'driftfix=slew' if $tdf;
3035
3036     if ($nokvm) {
3037         push @$machineFlags, 'accel=tcg';
3038     } else {
3039         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3040     }
3041
3042     if ($machine_type) {
3043         push @$machineFlags, "type=${machine_type}";
3044     }
3045
3046     if ($conf->{startdate}) {
3047         push @$rtcFlags, "base=$conf->{startdate}";
3048     } elsif ($useLocaltime) {
3049         push @$rtcFlags, 'base=localtime';
3050     }
3051
3052     my $cpu = $nokvm ? "qemu64" : "kvm64";
3053     if (my $cputype = $conf->{cpu}) {
3054         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3055             or die "Cannot parse cpu description: $cputype\n";
3056         $cpu = $cpuconf->{cputype};
3057         $kvm_off = 1 if $cpuconf->{hidden};
3058     }
3059
3060     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3061
3062     push @$cpuFlags , '-x2apic'
3063         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3064
3065     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3066
3067     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3068
3069     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3070
3071         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3072         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3073     }
3074
3075     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3076
3077     push @$cpuFlags, 'kvm=off' if $kvm_off;
3078
3079     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3080
3081     push @$cmd, '-cpu', $cpu;
3082
3083     my $memory = $conf->{memory} || $defaults->{memory};
3084     my $static_memory = 0;
3085     my $dimm_memory = 0;
3086
3087     if ($hotplug_features->{memory}) {
3088         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3089         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3090         $static_memory = $STATICMEM;
3091         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3092         $dimm_memory = $memory - $static_memory;
3093         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3094
3095     } else {
3096
3097         $static_memory = $memory;
3098         push @$cmd, '-m', $static_memory;
3099     }
3100
3101     if ($conf->{numa}) {
3102
3103         my $numa_totalmemory = undef;
3104         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3105             next if !$conf->{"numa$i"};
3106             my $numa = parse_numa($conf->{"numa$i"});
3107             next if !$numa;
3108             # memory
3109             die "missing numa node$i memory value\n" if !$numa->{memory};
3110             my $numa_memory = $numa->{memory};
3111             $numa_totalmemory += $numa_memory;
3112             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3113
3114             # cpus
3115             my $cpulists = $numa->{cpus};
3116             die "missing numa node$i cpus\n" if !defined($cpulists);
3117             my $cpus = join(',', map {
3118                 my ($start, $end) = @$_;
3119                 defined($end) ? "$start-$end" : $start
3120             } @$cpulists);
3121
3122             # hostnodes
3123             my $hostnodelists = $numa->{hostnodes};
3124             if (defined($hostnodelists)) {
3125                 my $hostnodes;
3126                 foreach my $hostnoderange (@$hostnodelists) {
3127                     my ($start, $end) = @$hostnoderange;
3128                     $hostnodes .= ',' if $hostnodes;
3129                     $hostnodes .= $start;
3130                     $hostnodes .= "-$end" if defined($end);
3131                     $end //= $start;
3132                     for (my $i = $start; $i <= $end; ++$i ) {
3133                         die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3134                     }
3135                 }
3136
3137                 # policy
3138                 my $policy = $numa->{policy};
3139                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3140                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3141             }
3142
3143             push @$cmd, '-object', $numa_object;
3144             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3145         }
3146
3147         die "total memory for NUMA nodes must be equal to vm static memory\n"
3148             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3149
3150         #if no custom tology, we split memory and cores across numa nodes
3151         if(!$numa_totalmemory) {
3152
3153             my $numa_memory = ($static_memory / $sockets) . "M";
3154
3155             for (my $i = 0; $i < $sockets; $i++)  {
3156
3157                 my $cpustart = ($cores * $i);
3158                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3159                 my $cpus = $cpustart;
3160                 $cpus .= "-$cpuend" if $cpuend;
3161
3162                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3163                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3164             }
3165         }
3166     }
3167
3168     if ($hotplug_features->{memory}) {
3169         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3170             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3171             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3172             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3173
3174             #if dimm_memory is not aligned to dimm map
3175             if($current_size > $memory) {
3176                  $conf->{memory} = $current_size;
3177                  PVE::QemuConfig->write_config($vmid, $conf);
3178             }
3179         });
3180     }
3181
3182     push @$cmd, '-S' if $conf->{freeze};
3183
3184     # set keyboard layout
3185     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3186     push @$cmd, '-k', $kb if $kb;
3187
3188     # enable sound
3189     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3190     #push @$cmd, '-soundhw', 'es1370';
3191     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3192
3193     if($conf->{agent}) {
3194         my $qgasocket = qmp_socket($vmid, 1);
3195         my $pciaddr = print_pci_addr("qga0", $bridges);
3196         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3197         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3198         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3199     }
3200
3201     my $spice_port;
3202
3203     if ($qxlnum) {
3204         if ($qxlnum > 1) {
3205             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3206                 for(my $i = 1; $i < $qxlnum; $i++){
3207                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3208                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3209                 }
3210             } else {
3211                 # assume other OS works like Linux
3212                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3213                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3214             }
3215         }
3216
3217         my $pciaddr = print_pci_addr("spice", $bridges);
3218
3219         my $nodename = PVE::INotify::nodename();
3220         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3221         $spice_port = PVE::Tools::next_spice_port($pfamily);
3222
3223         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3224
3225         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3226         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3227         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3228     }
3229
3230     # enable balloon by default, unless explicitly disabled
3231     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3232         $pciaddr = print_pci_addr("balloon0", $bridges);
3233         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3234     }
3235
3236     if ($conf->{watchdog}) {
3237         my $wdopts = parse_watchdog($conf->{watchdog});
3238         $pciaddr = print_pci_addr("watchdog", $bridges);
3239         my $watchdog = $wdopts->{model} || 'i6300esb';
3240         push @$devices, '-device', "$watchdog$pciaddr";
3241         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3242     }
3243
3244     my $vollist = [];
3245     my $scsicontroller = {};
3246     my $ahcicontroller = {};
3247     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3248
3249     # Add iscsi initiator name if available
3250     if (my $initiator = get_initiator_name()) {
3251         push @$devices, '-iscsi', "initiator-name=$initiator";
3252     }
3253
3254     foreach_drive($conf, sub {
3255         my ($ds, $drive) = @_;
3256
3257         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3258             push @$vollist, $drive->{file};
3259         }
3260
3261         $use_virtio = 1 if $ds =~ m/^virtio/;
3262
3263         if (drive_is_cdrom ($drive)) {
3264             if ($bootindex_hash->{d}) {
3265                 $drive->{bootindex} = $bootindex_hash->{d};
3266                 $bootindex_hash->{d} += 1;
3267             }
3268         } else {
3269             if ($bootindex_hash->{c}) {
3270                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3271                 $bootindex_hash->{c} += 1;
3272             }
3273         }
3274
3275         if($drive->{interface} eq 'virtio'){
3276            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3277         }
3278
3279         if ($drive->{interface} eq 'scsi') {
3280
3281             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3282
3283             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3284             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3285
3286             my $iothread = '';
3287             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3288                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3289                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3290             }
3291
3292             my $queues = '';
3293             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3294                 $queues = ",num_queues=$drive->{queues}";
3295             }
3296
3297             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3298             $scsicontroller->{$controller}=1;
3299         }
3300
3301         if ($drive->{interface} eq 'sata') {
3302            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3303            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3304            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3305            $ahcicontroller->{$controller}=1;
3306         }
3307
3308         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3309         push @$devices, '-drive',$drive_cmd;
3310         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3311     });
3312
3313     for (my $i = 0; $i < $MAX_NETS; $i++) {
3314          next if !$conf->{"net$i"};
3315          my $d = parse_net($conf->{"net$i"});
3316          next if !$d;
3317
3318          $use_virtio = 1 if $d->{model} eq 'virtio';
3319
3320          if ($bootindex_hash->{n}) {
3321             $d->{bootindex} = $bootindex_hash->{n};
3322             $bootindex_hash->{n} += 1;
3323          }
3324
3325          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3326          push @$devices, '-netdev', $netdevfull;
3327
3328          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3329          push @$devices, '-device', $netdevicefull;
3330     }
3331
3332     if (!$q35) {
3333         # add pci bridges
3334         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3335            $bridges->{1} = 1;
3336            $bridges->{2} = 1;
3337         }
3338
3339         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3340
3341         while (my ($k, $v) = each %$bridges) {
3342             $pciaddr = print_pci_addr("pci.$k");
3343             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3344         }
3345     }
3346
3347     # add custom args
3348     if ($conf->{args}) {
3349         my $aa = PVE::Tools::split_args($conf->{args});
3350         push @$cmd, @$aa;
3351     }
3352
3353     push @$cmd, @$devices;
3354     push @$cmd, '-rtc', join(',', @$rtcFlags)
3355         if scalar(@$rtcFlags);
3356     push @$cmd, '-machine', join(',', @$machineFlags)
3357         if scalar(@$machineFlags);
3358     push @$cmd, '-global', join(',', @$globalFlags)
3359         if scalar(@$globalFlags);
3360
3361     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3362 }
3363
3364 sub vnc_socket {
3365     my ($vmid) = @_;
3366     return "${var_run_tmpdir}/$vmid.vnc";
3367 }
3368
3369 sub spice_port {
3370     my ($vmid) = @_;
3371
3372     my $res = vm_mon_cmd($vmid, 'query-spice');
3373
3374     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3375 }
3376
3377 sub qmp_socket {
3378     my ($vmid, $qga) = @_;
3379     my $sockettype = $qga ? 'qga' : 'qmp';
3380     return "${var_run_tmpdir}/$vmid.$sockettype";
3381 }
3382
3383 sub pidfile_name {
3384     my ($vmid) = @_;
3385     return "${var_run_tmpdir}/$vmid.pid";
3386 }
3387
3388 sub vm_devices_list {
3389     my ($vmid) = @_;
3390
3391     my $res = vm_mon_cmd($vmid, 'query-pci');
3392     my $devices = {};
3393     foreach my $pcibus (@$res) {
3394         foreach my $device (@{$pcibus->{devices}}) {
3395             next if !$device->{'qdev_id'};
3396             if ($device->{'pci_bridge'}) {
3397                 $devices->{$device->{'qdev_id'}} = 1;
3398                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3399                     next if !$bridge_device->{'qdev_id'};
3400                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3401                     $devices->{$device->{'qdev_id'}}++;
3402                 }
3403             } else {
3404                 $devices->{$device->{'qdev_id'}} = 1;
3405             }
3406         }
3407     }
3408
3409     my $resblock = vm_mon_cmd($vmid, 'query-block');
3410     foreach my $block (@$resblock) {
3411         if($block->{device} =~ m/^drive-(\S+)/){
3412                 $devices->{$1} = 1;
3413         }
3414     }
3415
3416     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3417     foreach my $mice (@$resmice) {
3418         if ($mice->{name} eq 'QEMU HID Tablet') {
3419             $devices->{tablet} = 1;
3420             last;
3421         }
3422     }
3423
3424     return $devices;
3425 }
3426
3427 sub vm_deviceplug {
3428     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3429
3430     my $q35 = machine_type_is_q35($conf);
3431
3432     my $devices_list = vm_devices_list($vmid);
3433     return 1 if defined($devices_list->{$deviceid});
3434
3435     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3436
3437     if ($deviceid eq 'tablet') {
3438
3439         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3440
3441     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3442
3443         qemu_iothread_add($vmid, $deviceid, $device);
3444
3445         qemu_driveadd($storecfg, $vmid, $device);
3446         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3447
3448         qemu_deviceadd($vmid, $devicefull);
3449         eval { qemu_deviceaddverify($vmid, $deviceid); };
3450         if (my $err = $@) {
3451             eval { qemu_drivedel($vmid, $deviceid); };
3452             warn $@ if $@;
3453             die $err;
3454         }
3455
3456     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3457
3458
3459         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3460         my $pciaddr = print_pci_addr($deviceid);
3461         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3462
3463         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3464
3465         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3466             qemu_iothread_add($vmid, $deviceid, $device);
3467             $devicefull .= ",iothread=iothread-$deviceid";
3468         }
3469
3470         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3471             $devicefull .= ",num_queues=$device->{queues}";
3472         }
3473
3474         qemu_deviceadd($vmid, $devicefull);
3475         qemu_deviceaddverify($vmid, $deviceid);
3476
3477     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3478
3479         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3480         qemu_driveadd($storecfg, $vmid, $device);
3481
3482         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3483         eval { qemu_deviceadd($vmid, $devicefull); };
3484         if (my $err = $@) {
3485             eval { qemu_drivedel($vmid, $deviceid); };
3486             warn $@ if $@;
3487             die $err;
3488         }
3489
3490     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3491
3492         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3493
3494         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3495         my $use_old_bios_files = undef;
3496         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3497
3498         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3499         qemu_deviceadd($vmid, $netdevicefull);
3500         eval { qemu_deviceaddverify($vmid, $deviceid); };
3501         if (my $err = $@) {
3502             eval { qemu_netdevdel($vmid, $deviceid); };
3503             warn $@ if $@;
3504             die $err;
3505         }
3506
3507     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3508
3509         my $bridgeid = $2;
3510         my $pciaddr = print_pci_addr($deviceid);
3511         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3512
3513         qemu_deviceadd($vmid, $devicefull);
3514         qemu_deviceaddverify($vmid, $deviceid);
3515
3516     } else {
3517         die "can't hotplug device '$deviceid'\n";
3518     }
3519
3520     return 1;
3521 }
3522
3523 # fixme: this should raise exceptions on error!
3524 sub vm_deviceunplug {
3525     my ($vmid, $conf, $deviceid) = @_;
3526
3527     my $devices_list = vm_devices_list($vmid);
3528     return 1 if !defined($devices_list->{$deviceid});
3529
3530     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3531
3532     if ($deviceid eq 'tablet') {
3533
3534         qemu_devicedel($vmid, $deviceid);
3535
3536     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3537
3538         qemu_devicedel($vmid, $deviceid);
3539         qemu_devicedelverify($vmid, $deviceid);
3540         qemu_drivedel($vmid, $deviceid);
3541         qemu_iothread_del($conf, $vmid, $deviceid);
3542
3543     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3544
3545         qemu_devicedel($vmid, $deviceid);
3546         qemu_devicedelverify($vmid, $deviceid);
3547         qemu_iothread_del($conf, $vmid, $deviceid);
3548
3549     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3550
3551         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3552         my $device = parse_drive($deviceid, $conf->{$deviceid});
3553         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3554
3555         qemu_devicedel($vmid, $deviceid);
3556         qemu_drivedel($vmid, $deviceid);
3557         qemu_deletescsihw($conf, $vmid, $deviceid);
3558
3559     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3560
3561         qemu_devicedel($vmid, $deviceid);
3562         qemu_devicedelverify($vmid, $deviceid);
3563         qemu_netdevdel($vmid, $deviceid);
3564
3565     } else {
3566         die "can't unplug device '$deviceid'\n";
3567     }
3568
3569     return 1;
3570 }
3571
3572 sub qemu_deviceadd {
3573     my ($vmid, $devicefull) = @_;
3574
3575     $devicefull = "driver=".$devicefull;
3576     my %options =  split(/[=,]/, $devicefull);
3577
3578     vm_mon_cmd($vmid, "device_add" , %options);
3579 }
3580
3581 sub qemu_devicedel {
3582     my ($vmid, $deviceid) = @_;
3583
3584     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3585 }
3586
3587 sub qemu_iothread_add {
3588     my($vmid, $deviceid, $device) = @_;
3589
3590     if ($device->{iothread}) {
3591         my $iothreads = vm_iothreads_list($vmid);
3592         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3593     }
3594 }
3595
3596 sub qemu_iothread_del {
3597     my($conf, $vmid, $deviceid) = @_;
3598
3599     my $device = parse_drive($deviceid, $conf->{$deviceid});
3600     if ($device->{iothread}) {
3601         my $iothreads = vm_iothreads_list($vmid);
3602         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3603     }
3604 }
3605
3606 sub qemu_objectadd {
3607     my($vmid, $objectid, $qomtype) = @_;
3608
3609     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3610
3611     return 1;
3612 }
3613
3614 sub qemu_objectdel {
3615     my($vmid, $objectid) = @_;
3616
3617     vm_mon_cmd($vmid, "object-del", id => $objectid);
3618
3619     return 1;
3620 }
3621
3622 sub qemu_driveadd {
3623     my ($storecfg, $vmid, $device) = @_;
3624
3625     my $drive = print_drive_full($storecfg, $vmid, $device);
3626     $drive =~ s/\\/\\\\/g;
3627     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3628
3629     # If the command succeeds qemu prints: "OK"
3630     return 1 if $ret =~ m/OK/s;
3631
3632     die "adding drive failed: $ret\n";
3633 }
3634
3635 sub qemu_drivedel {
3636     my($vmid, $deviceid) = @_;
3637
3638     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3639     $ret =~ s/^\s+//;
3640
3641     return 1 if $ret eq "";
3642
3643     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3644     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3645
3646     die "deleting drive $deviceid failed : $ret\n";
3647 }
3648
3649 sub qemu_deviceaddverify {
3650     my ($vmid, $deviceid) = @_;
3651
3652     for (my $i = 0; $i <= 5; $i++) {
3653          my $devices_list = vm_devices_list($vmid);
3654          return 1 if defined($devices_list->{$deviceid});
3655          sleep 1;
3656     }
3657
3658     die "error on hotplug device '$deviceid'\n";
3659 }
3660
3661
3662 sub qemu_devicedelverify {
3663     my ($vmid, $deviceid) = @_;
3664
3665     # need to verify that the device is correctly removed as device_del
3666     # is async and empty return is not reliable
3667
3668     for (my $i = 0; $i <= 5; $i++) {
3669          my $devices_list = vm_devices_list($vmid);
3670          return 1 if !defined($devices_list->{$deviceid});
3671          sleep 1;
3672     }
3673
3674     die "error on hot-unplugging device '$deviceid'\n";
3675 }
3676
3677 sub qemu_findorcreatescsihw {
3678     my ($storecfg, $conf, $vmid, $device) = @_;
3679
3680     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3681
3682     my $scsihwid="$controller_prefix$controller";
3683     my $devices_list = vm_devices_list($vmid);
3684
3685     if(!defined($devices_list->{$scsihwid})) {
3686         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3687     }
3688
3689     return 1;
3690 }
3691
3692 sub qemu_deletescsihw {
3693     my ($conf, $vmid, $opt) = @_;
3694
3695     my $device = parse_drive($opt, $conf->{$opt});
3696
3697     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3698         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3699         return 1;
3700     }
3701
3702     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3703
3704     my $devices_list = vm_devices_list($vmid);
3705     foreach my $opt (keys %{$devices_list}) {
3706         if (PVE::QemuServer::is_valid_drivename($opt)) {
3707             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3708             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3709                 return 1;
3710             }
3711         }
3712     }
3713
3714     my $scsihwid="scsihw$controller";
3715
3716     vm_deviceunplug($vmid, $conf, $scsihwid);
3717
3718     return 1;
3719 }
3720
3721 sub qemu_add_pci_bridge {
3722     my ($storecfg, $conf, $vmid, $device) = @_;
3723
3724     my $bridges = {};
3725
3726     my $bridgeid;
3727
3728     print_pci_addr($device, $bridges);
3729
3730     while (my ($k, $v) = each %$bridges) {
3731         $bridgeid = $k;
3732     }
3733     return 1 if !defined($bridgeid) || $bridgeid < 1;
3734
3735     my $bridge = "pci.$bridgeid";
3736     my $devices_list = vm_devices_list($vmid);
3737
3738     if (!defined($devices_list->{$bridge})) {
3739         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3740     }
3741
3742     return 1;
3743 }
3744
3745 sub qemu_set_link_status {
3746     my ($vmid, $device, $up) = @_;
3747
3748     vm_mon_cmd($vmid, "set_link", name => $device,
3749                up => $up ? JSON::true : JSON::false);
3750 }
3751
3752 sub qemu_netdevadd {
3753     my ($vmid, $conf, $device, $deviceid) = @_;
3754
3755     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3756     my %options =  split(/[=,]/, $netdev);
3757
3758     vm_mon_cmd($vmid, "netdev_add",  %options);
3759     return 1;
3760 }
3761
3762 sub qemu_netdevdel {
3763     my ($vmid, $deviceid) = @_;
3764
3765     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3766 }
3767
3768 sub qemu_cpu_hotplug {
3769     my ($vmid, $conf, $vcpus) = @_;
3770
3771     my $sockets = 1;
3772     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3773     $sockets = $conf->{sockets} if  $conf->{sockets};
3774     my $cores = $conf->{cores} || 1;
3775     my $maxcpus = $sockets * $cores;
3776
3777     $vcpus = $maxcpus if !$vcpus;
3778
3779     die "you can't add more vcpus than maxcpus\n"
3780         if $vcpus > $maxcpus;
3781
3782     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3783     die "online cpu unplug is not yet possible\n"
3784         if $vcpus < $currentvcpus;
3785
3786     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3787     die "vcpus in running vm is different than configuration\n"
3788         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3789
3790     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3791         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3792     }
3793 }
3794
3795 sub qemu_memory_hotplug {
3796     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3797
3798     return $value if !check_running($vmid);
3799
3800     my $memory = $conf->{memory} || $defaults->{memory};
3801     $value = $defaults->{memory} if !$value;
3802     return $value if $value == $memory;
3803
3804     my $static_memory = $STATICMEM;
3805     my $dimm_memory = $memory - $static_memory;
3806
3807     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3808     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3809
3810
3811     my $sockets = 1;
3812     $sockets = $conf->{sockets} if $conf->{sockets};
3813
3814     if($value > $memory) {
3815
3816         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3817             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3818
3819                 return if $current_size <= $conf->{memory};
3820
3821                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3822                 if (my $err = $@) {
3823                     eval { qemu_objectdel($vmid, "mem-$name"); };
3824                     die $err;
3825                 }
3826
3827                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3828                 if (my $err = $@) {
3829                     eval { qemu_objectdel($vmid, "mem-$name"); };
3830                     die $err;
3831                 }
3832                 #update conf after each succesful module hotplug
3833                 $conf->{memory} = $current_size;
3834                 PVE::QemuConfig->write_config($vmid, $conf);
3835         });
3836
3837     } else {
3838
3839         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3840             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3841
3842                 return if $current_size >= $conf->{memory};
3843                 print "try to unplug memory dimm $name\n";
3844
3845                 my $retry = 0;
3846                 while (1) {
3847                     eval { qemu_devicedel($vmid, $name) };
3848                     sleep 3;
3849                     my $dimm_list = qemu_dimm_list($vmid);
3850                     last if !$dimm_list->{$name};
3851                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3852                     $retry++;
3853                 }
3854
3855                 #update conf after each succesful module unplug
3856                 $conf->{memory} = $current_size;
3857
3858                 eval { qemu_objectdel($vmid, "mem-$name"); };
3859                 PVE::QemuConfig->write_config($vmid, $conf);
3860         });
3861     }
3862 }
3863
3864 sub qemu_dimm_list {
3865     my ($vmid) = @_;
3866
3867     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3868     my $dimms = {};
3869
3870     foreach my $dimm (@$dimmarray) {
3871
3872         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3873         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3874         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3875         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3876         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3877     }
3878     return $dimms;
3879 }
3880
3881 sub qemu_block_set_io_throttle {
3882     my ($vmid, $deviceid,
3883         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3884         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3885
3886     return if !check_running($vmid) ;
3887
3888     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3889         bps => int($bps),
3890         bps_rd => int($bps_rd),
3891         bps_wr => int($bps_wr),
3892         iops => int($iops),
3893         iops_rd => int($iops_rd),
3894         iops_wr => int($iops_wr),
3895         bps_max => int($bps_max),
3896         bps_rd_max => int($bps_rd_max),
3897         bps_wr_max => int($bps_wr_max),
3898         iops_max => int($iops_max),
3899         iops_rd_max => int($iops_rd_max),
3900         iops_wr_max => int($iops_wr_max)
3901     );
3902
3903 }
3904
3905 # old code, only used to shutdown old VM after update
3906 sub __read_avail {
3907     my ($fh, $timeout) = @_;
3908
3909     my $sel = new IO::Select;
3910     $sel->add($fh);
3911
3912     my $res = '';
3913     my $buf;
3914
3915     my @ready;
3916     while (scalar (@ready = $sel->can_read($timeout))) {
3917         my $count;
3918         if ($count = $fh->sysread($buf, 8192)) {
3919             if ($buf =~ /^(.*)\(qemu\) $/s) {
3920                 $res .= $1;
3921                 last;
3922             } else {
3923                 $res .= $buf;
3924             }
3925         } else {
3926             if (!defined($count)) {
3927                 die "$!\n";
3928             }
3929             last;
3930         }
3931     }
3932
3933     die "monitor read timeout\n" if !scalar(@ready);
3934
3935     return $res;
3936 }
3937
3938 # old code, only used to shutdown old VM after update
3939 sub vm_monitor_command {
3940     my ($vmid, $cmdstr, $nocheck) = @_;
3941
3942     my $res;
3943
3944     eval {
3945         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3946
3947         my $sname = "${var_run_tmpdir}/$vmid.mon";
3948
3949         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3950             die "unable to connect to VM $vmid socket - $!\n";
3951
3952         my $timeout = 3;
3953
3954         # hack: migrate sometime blocks the monitor (when migrate_downtime
3955         # is set)
3956         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3957             $timeout = 60*60; # 1 hour
3958         }
3959
3960         # read banner;
3961         my $data = __read_avail($sock, $timeout);
3962
3963         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3964             die "got unexpected qemu monitor banner\n";
3965         }
3966
3967         my $sel = new IO::Select;
3968         $sel->add($sock);
3969
3970         if (!scalar(my @ready = $sel->can_write($timeout))) {
3971             die "monitor write error - timeout";
3972         }
3973
3974         my $fullcmd = "$cmdstr\r";
3975
3976         # syslog('info', "VM $vmid monitor command: $cmdstr");
3977
3978         my $b;
3979         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3980             die "monitor write error - $!";
3981         }
3982
3983         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3984
3985         $timeout = 20;
3986
3987         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3988             $timeout = 60*60; # 1 hour
3989         } elsif ($cmdstr =~ m/^(eject|change)/) {
3990             $timeout = 60; # note: cdrom mount command is slow
3991         }
3992         if ($res = __read_avail($sock, $timeout)) {
3993
3994             my @lines = split("\r?\n", $res);
3995
3996             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3997
3998             $res = join("\n", @lines);
3999             $res .= "\n";
4000         }
4001     };
4002
4003     my $err = $@;
4004
4005     if ($err) {
4006         syslog("err", "VM $vmid monitor command failed - $err");
4007         die $err;
4008     }
4009
4010     return $res;
4011 }
4012
4013 sub qemu_block_resize {
4014     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4015
4016     my $running = check_running($vmid);
4017
4018     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4019
4020     return if !$running;
4021
4022     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4023
4024 }
4025
4026 sub qemu_volume_snapshot {
4027     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4028
4029     my $running = check_running($vmid);
4030
4031     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4032         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4033     } else {
4034         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4035     }
4036 }
4037
4038 sub qemu_volume_snapshot_delete {
4039     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4040
4041     my $running = check_running($vmid);
4042
4043     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4044
4045     return if !$running;
4046
4047     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4048 }
4049
4050 sub set_migration_caps {
4051     my ($vmid) = @_;
4052
4053     my $cap_ref = [];
4054
4055     my $enabled_cap = {
4056         "auto-converge" => 1,
4057         "xbzrle" => 1,
4058         "x-rdma-pin-all" => 0,
4059         "zero-blocks" => 0,
4060         "compress" => 0
4061     };
4062
4063     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4064
4065     for my $supported_capability (@$supported_capabilities) {
4066         push @$cap_ref, {
4067             capability => $supported_capability->{capability},
4068             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4069         };
4070     }
4071
4072     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4073 }
4074
4075 my $fast_plug_option = {
4076     'lock' => 1,
4077     'name' => 1,
4078     'onboot' => 1,
4079     'shares' => 1,
4080     'startup' => 1,
4081     'description' => 1,
4082 };
4083
4084 # hotplug changes in [PENDING]
4085 # $selection hash can be used to only apply specified options, for
4086 # example: { cores => 1 } (only apply changed 'cores')
4087 # $errors ref is used to return error messages
4088 sub vmconfig_hotplug_pending {
4089     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4090
4091     my $defaults = load_defaults();
4092
4093     # commit values which do not have any impact on running VM first
4094     # Note: those option cannot raise errors, we we do not care about
4095     # $selection and always apply them.
4096
4097     my $add_error = sub {
4098         my ($opt, $msg) = @_;
4099         $errors->{$opt} = "hotplug problem - $msg";
4100     };
4101
4102     my $changes = 0;
4103     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4104         if ($fast_plug_option->{$opt}) {
4105             $conf->{$opt} = $conf->{pending}->{$opt};
4106             delete $conf->{pending}->{$opt};
4107             $changes = 1;
4108         }
4109     }
4110
4111     if ($changes) {
4112         PVE::QemuConfig->write_config($vmid, $conf);
4113         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4114     }
4115
4116     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4117
4118     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4119     while (my ($opt, $force) = each %$pending_delete_hash) {
4120         next if $selection && !$selection->{$opt};
4121         eval {
4122             if ($opt eq 'hotplug') {
4123                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4124             } elsif ($opt eq 'tablet') {
4125                 die "skip\n" if !$hotplug_features->{usb};
4126                 if ($defaults->{tablet}) {
4127                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4128                 } else {
4129                     vm_deviceunplug($vmid, $conf, $opt);
4130                 }
4131             } elsif ($opt eq 'vcpus') {
4132                 die "skip\n" if !$hotplug_features->{cpu};
4133                 qemu_cpu_hotplug($vmid, $conf, undef);
4134             } elsif ($opt eq 'balloon') {
4135                 # enable balloon device is not hotpluggable
4136                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4137             } elsif ($fast_plug_option->{$opt}) {
4138                 # do nothing
4139             } elsif ($opt =~ m/^net(\d+)$/) {
4140                 die "skip\n" if !$hotplug_features->{network};
4141                 vm_deviceunplug($vmid, $conf, $opt);
4142             } elsif (is_valid_drivename($opt)) {
4143                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4144                 vm_deviceunplug($vmid, $conf, $opt);
4145                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4146             } elsif ($opt =~ m/^memory$/) {
4147                 die "skip\n" if !$hotplug_features->{memory};
4148                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4149             } elsif ($opt eq 'cpuunits') {
4150                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4151             } elsif ($opt eq 'cpulimit') {
4152                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4153             } else {
4154                 die "skip\n";
4155             }
4156         };
4157         if (my $err = $@) {
4158             &$add_error($opt, $err) if $err ne "skip\n";
4159         } else {
4160             # save new config if hotplug was successful
4161             delete $conf->{$opt};
4162             vmconfig_undelete_pending_option($conf, $opt);
4163             PVE::QemuConfig->write_config($vmid, $conf);
4164             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4165         }
4166     }
4167
4168     foreach my $opt (keys %{$conf->{pending}}) {
4169         next if $selection && !$selection->{$opt};
4170         my $value = $conf->{pending}->{$opt};
4171         eval {
4172             if ($opt eq 'hotplug') {
4173                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4174             } elsif ($opt eq 'tablet') {
4175                 die "skip\n" if !$hotplug_features->{usb};
4176                 if ($value == 1) {
4177                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4178                 } elsif ($value == 0) {
4179                     vm_deviceunplug($vmid, $conf, $opt);
4180                 }
4181             } elsif ($opt eq 'vcpus') {
4182                 die "skip\n" if !$hotplug_features->{cpu};
4183                 qemu_cpu_hotplug($vmid, $conf, $value);
4184             } elsif ($opt eq 'balloon') {
4185                 # enable/disable balloning device is not hotpluggable
4186                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4187                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4188                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4189
4190                 # allow manual ballooning if shares is set to zero
4191                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4192                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4193                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4194                 }
4195             } elsif ($opt =~ m/^net(\d+)$/) {
4196                 # some changes can be done without hotplug
4197                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4198                                     $vmid, $opt, $value);
4199             } elsif (is_valid_drivename($opt)) {
4200                 # some changes can be done without hotplug
4201                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4202                                      $vmid, $opt, $value, 1);
4203             } elsif ($opt =~ m/^memory$/) { #dimms
4204                 die "skip\n" if !$hotplug_features->{memory};
4205                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4206             } elsif ($opt eq 'cpuunits') {
4207                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4208             } elsif ($opt eq 'cpulimit') {
4209                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4210                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4211             } else {
4212                 die "skip\n";  # skip non-hot-pluggable options
4213             }
4214         };
4215         if (my $err = $@) {
4216             &$add_error($opt, $err) if $err ne "skip\n";
4217         } else {
4218             # save new config if hotplug was successful
4219             $conf->{$opt} = $value;
4220             delete $conf->{pending}->{$opt};
4221             PVE::QemuConfig->write_config($vmid, $conf);
4222             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4223         }
4224     }
4225 }
4226
4227 sub try_deallocate_drive {
4228     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4229
4230     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4231         my $volid = $drive->{file};
4232         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4233             my $sid = PVE::Storage::parse_volume_id($volid);
4234             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4235
4236             # check if the disk is really unused
4237             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4238                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4239             PVE::Storage::vdisk_free($storecfg, $volid);
4240             return 1;
4241         } else {
4242             # If vm is not owner of this disk remove from config
4243             return 1;
4244         }
4245     }
4246
4247     return undef;
4248 }
4249
4250 sub vmconfig_delete_or_detach_drive {
4251     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4252
4253     my $drive = parse_drive($opt, $conf->{$opt});
4254
4255     my $rpcenv = PVE::RPCEnvironment::get();
4256     my $authuser = $rpcenv->get_user();
4257
4258     if ($force) {
4259         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4260         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4261     } else {
4262         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4263     }
4264 }
4265
4266 sub vmconfig_apply_pending {
4267     my ($vmid, $conf, $storecfg) = @_;
4268
4269     # cold plug
4270
4271     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4272     while (my ($opt, $force) = each %$pending_delete_hash) {
4273         die "internal error" if $opt =~ m/^unused/;
4274         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4275         if (!defined($conf->{$opt})) {
4276             vmconfig_undelete_pending_option($conf, $opt);
4277             PVE::QemuConfig->write_config($vmid, $conf);
4278         } elsif (is_valid_drivename($opt)) {
4279             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4280             vmconfig_undelete_pending_option($conf, $opt);
4281             delete $conf->{$opt};
4282             PVE::QemuConfig->write_config($vmid, $conf);
4283         } else {
4284             vmconfig_undelete_pending_option($conf, $opt);
4285             delete $conf->{$opt};
4286             PVE::QemuConfig->write_config($vmid, $conf);
4287         }
4288     }
4289
4290     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4291
4292     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4293         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4294
4295         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4296             # skip if nothing changed
4297         } elsif (is_valid_drivename($opt)) {
4298             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4299                 if defined($conf->{$opt});
4300             $conf->{$opt} = $conf->{pending}->{$opt};
4301         } else {
4302             $conf->{$opt} = $conf->{pending}->{$opt};
4303         }
4304
4305         delete $conf->{pending}->{$opt};
4306         PVE::QemuConfig->write_config($vmid, $conf);
4307     }
4308 }
4309
4310 my $safe_num_ne = sub {
4311     my ($a, $b) = @_;
4312
4313     return 0 if !defined($a) && !defined($b);
4314     return 1 if !defined($a);
4315     return 1 if !defined($b);
4316
4317     return $a != $b;
4318 };
4319
4320 my $safe_string_ne = sub {
4321     my ($a, $b) = @_;
4322
4323     return 0 if !defined($a) && !defined($b);
4324     return 1 if !defined($a);
4325     return 1 if !defined($b);
4326
4327     return $a ne $b;
4328 };
4329
4330 sub vmconfig_update_net {
4331     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4332
4333     my $newnet = parse_net($value);
4334
4335     if ($conf->{$opt}) {
4336         my $oldnet = parse_net($conf->{$opt});
4337
4338         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4339             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4340             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4341             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4342
4343             # for non online change, we try to hot-unplug
4344             die "skip\n" if !$hotplug;
4345             vm_deviceunplug($vmid, $conf, $opt);
4346         } else {
4347
4348             die "internal error" if $opt !~ m/net(\d+)/;
4349             my $iface = "tap${vmid}i$1";
4350
4351             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4352                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4353                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4354                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4355                 PVE::Network::tap_unplug($iface);
4356                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4357             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4358                 # Rate can be applied on its own but any change above needs to
4359                 # include the rate in tap_plug since OVS resets everything.
4360                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4361             }
4362
4363             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4364                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4365             }
4366
4367             return 1;
4368         }
4369     }
4370
4371     if ($hotplug) {
4372         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4373     } else {
4374         die "skip\n";
4375     }
4376 }
4377
4378 sub vmconfig_update_disk {
4379     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4380
4381     # fixme: do we need force?
4382
4383     my $drive = parse_drive($opt, $value);
4384
4385     if ($conf->{$opt}) {
4386
4387         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4388
4389             my $media = $drive->{media} || 'disk';
4390             my $oldmedia = $old_drive->{media} || 'disk';
4391             die "unable to change media type\n" if $media ne $oldmedia;
4392
4393             if (!drive_is_cdrom($old_drive)) {
4394
4395                 if ($drive->{file} ne $old_drive->{file}) {
4396
4397                     die "skip\n" if !$hotplug;
4398
4399                     # unplug and register as unused
4400                     vm_deviceunplug($vmid, $conf, $opt);
4401                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4402
4403                 } else {
4404                     # update existing disk
4405
4406                     # skip non hotpluggable value
4407                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4408                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4409                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4410                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4411                         die "skip\n";
4412                     }
4413
4414                     # apply throttle
4415                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4416                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4417                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4418                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4419                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4420                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4421                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4422                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4423                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4424                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4425                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4426                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4427
4428                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4429                                                    ($drive->{mbps} || 0)*1024*1024,
4430                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4431                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4432                                                    $drive->{iops} || 0,
4433                                                    $drive->{iops_rd} || 0,
4434                                                    $drive->{iops_wr} || 0,
4435                                                    ($drive->{mbps_max} || 0)*1024*1024,
4436                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4437                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4438                                                    $drive->{iops_max} || 0,
4439                                                    $drive->{iops_rd_max} || 0,
4440                                                    $drive->{iops_wr_max} || 0);
4441
4442                     }
4443
4444                     return 1;
4445                 }
4446
4447             } else { # cdrom
4448
4449                 if ($drive->{file} eq 'none') {
4450                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4451                 } else {
4452                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4453                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4454                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4455                 }
4456
4457                 return 1;
4458             }
4459         }
4460     }
4461
4462     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4463     # hotplug new disks
4464     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4465     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4466 }
4467
4468 sub vm_start {
4469     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4470         $forcemachine, $spice_ticket) = @_;
4471
4472     PVE::QemuConfig->lock_config($vmid, sub {
4473         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4474
4475         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4476
4477         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4478
4479         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4480
4481         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4482             vmconfig_apply_pending($vmid, $conf, $storecfg);
4483             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4484         }
4485
4486         my $defaults = load_defaults();
4487
4488         # set environment variable useful inside network script
4489         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4490
4491         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4492
4493         my $migrate_port = 0;
4494         my $migrate_uri;
4495         if ($statefile) {
4496             if ($statefile eq 'tcp') {
4497                 my $localip = "localhost";
4498                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4499                 my $nodename = PVE::INotify::nodename();
4500                 if ($datacenterconf->{migration_unsecure}) {
4501                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4502                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4503                 }
4504                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4505                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4506                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4507                 push @$cmd, '-incoming', $migrate_uri;
4508                 push @$cmd, '-S';
4509             } else {
4510                 push @$cmd, '-loadstate', $statefile;
4511             }
4512         } elsif ($paused) {
4513             push @$cmd, '-S';
4514         }
4515
4516         # host pci devices
4517         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4518           my $d = parse_hostpci($conf->{"hostpci$i"});
4519           next if !$d;
4520           my $pcidevices = $d->{pciid};
4521           foreach my $pcidevice (@$pcidevices) {
4522                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4523
4524                 my $info = pci_device_info("0000:$pciid");
4525                 die "IOMMU not present\n" if !check_iommu_support();
4526                 die "no pci device info for device '$pciid'\n" if !$info;
4527                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4528                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4529           }
4530         }
4531
4532         PVE::Storage::activate_volumes($storecfg, $vollist);
4533
4534         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4535                     umask => 0077); };
4536
4537         if (my $err = $@) {
4538             # deactivate volumes if start fails
4539             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4540             die "start failed: $err";
4541         }
4542
4543         print "migration listens on $migrate_uri\n" if $migrate_uri;
4544
4545         if ($statefile && $statefile ne 'tcp')  {
4546             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4547             warn $@ if $@;
4548         }
4549
4550         if ($migratedfrom) {
4551
4552             eval {
4553                 set_migration_caps($vmid);
4554             };
4555             warn $@ if $@;
4556
4557             if ($spice_port) {
4558                 print "spice listens on port $spice_port\n";
4559                 if ($spice_ticket) {
4560                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4561                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4562                 }
4563             }
4564
4565         } else {
4566
4567             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4568                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4569                     if $conf->{balloon};
4570             }
4571
4572             foreach my $opt (keys %$conf) {
4573                 next if $opt !~  m/^net\d+$/;
4574                 my $nicconf = parse_net($conf->{$opt});
4575                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4576             }
4577         }
4578
4579         vm_mon_cmd_nocheck($vmid, 'qom-set',
4580                     path => "machine/peripheral/balloon0",
4581                     property => "guest-stats-polling-interval",
4582                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4583
4584     });
4585 }
4586
4587 sub vm_mon_cmd {
4588     my ($vmid, $execute, %params) = @_;
4589
4590     my $cmd = { execute => $execute, arguments => \%params };
4591     vm_qmp_command($vmid, $cmd);
4592 }
4593
4594 sub vm_mon_cmd_nocheck {
4595     my ($vmid, $execute, %params) = @_;
4596
4597     my $cmd = { execute => $execute, arguments => \%params };
4598     vm_qmp_command($vmid, $cmd, 1);
4599 }
4600
4601 sub vm_qmp_command {
4602     my ($vmid, $cmd, $nocheck) = @_;
4603
4604     my $res;
4605
4606     my $timeout;
4607     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4608         $timeout = $cmd->{arguments}->{timeout};
4609         delete $cmd->{arguments}->{timeout};
4610     }
4611
4612     eval {
4613         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4614         my $sname = qmp_socket($vmid);
4615         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4616             my $qmpclient = PVE::QMPClient->new();
4617
4618             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4619         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4620             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4621                 if scalar(%{$cmd->{arguments}});
4622             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4623         } else {
4624             die "unable to open monitor socket\n";
4625         }
4626     };
4627     if (my $err = $@) {
4628         syslog("err", "VM $vmid qmp command failed - $err");
4629         die $err;
4630     }
4631
4632     return $res;
4633 }
4634
4635 sub vm_human_monitor_command {
4636     my ($vmid, $cmdline) = @_;
4637
4638     my $res;
4639
4640     my $cmd = {
4641         execute => 'human-monitor-command',
4642         arguments => { 'command-line' => $cmdline},
4643     };
4644
4645     return vm_qmp_command($vmid, $cmd);
4646 }
4647
4648 sub vm_commandline {
4649     my ($storecfg, $vmid) = @_;
4650
4651     my $conf = PVE::QemuConfig->load_config($vmid);
4652
4653     my $defaults = load_defaults();
4654
4655     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4656
4657     return join(' ', @$cmd);
4658 }
4659
4660 sub vm_reset {
4661     my ($vmid, $skiplock) = @_;
4662
4663     PVE::QemuConfig->lock_config($vmid, sub {
4664
4665         my $conf = PVE::QemuConfig->load_config($vmid);
4666
4667         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4668
4669         vm_mon_cmd($vmid, "system_reset");
4670     });
4671 }
4672
4673 sub get_vm_volumes {
4674     my ($conf) = @_;
4675
4676     my $vollist = [];
4677     foreach_volid($conf, sub {
4678         my ($volid, $is_cdrom) = @_;
4679
4680         return if $volid =~ m|^/|;
4681
4682         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4683         return if !$sid;
4684
4685         push @$vollist, $volid;
4686     });
4687
4688     return $vollist;
4689 }
4690
4691 sub vm_stop_cleanup {
4692     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4693
4694     eval {
4695
4696         if (!$keepActive) {
4697             my $vollist = get_vm_volumes($conf);
4698             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4699         }
4700
4701         foreach my $ext (qw(mon qmp pid vnc qga)) {
4702             unlink "/var/run/qemu-server/${vmid}.$ext";
4703         }
4704
4705         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4706     };
4707     warn $@ if $@; # avoid errors - just warn
4708 }
4709
4710 # Note: use $nockeck to skip tests if VM configuration file exists.
4711 # We need that when migration VMs to other nodes (files already moved)
4712 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4713 sub vm_stop {
4714     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4715
4716     $force = 1 if !defined($force) && !$shutdown;
4717
4718     if ($migratedfrom){
4719         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4720         kill 15, $pid if $pid;
4721         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4722         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4723         return;
4724     }
4725
4726     PVE::QemuConfig->lock_config($vmid, sub {
4727
4728         my $pid = check_running($vmid, $nocheck);
4729         return if !$pid;
4730
4731         my $conf;
4732         if (!$nocheck) {
4733             $conf = PVE::QemuConfig->load_config($vmid);
4734             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4735             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4736                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4737                 $timeout = $opts->{down} if $opts->{down};
4738             }
4739         }
4740
4741         $timeout = 60 if !defined($timeout);
4742
4743         eval {
4744             if ($shutdown) {
4745                 if (defined($conf) && $conf->{agent}) {
4746                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4747                 } else {
4748                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4749                 }
4750             } else {
4751                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4752             }
4753         };
4754         my $err = $@;
4755
4756         if (!$err) {
4757             my $count = 0;
4758             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4759                 $count++;
4760                 sleep 1;
4761             }
4762
4763             if ($count >= $timeout) {
4764                 if ($force) {
4765                     warn "VM still running - terminating now with SIGTERM\n";
4766                     kill 15, $pid;
4767                 } else {
4768                     die "VM quit/powerdown failed - got timeout\n";
4769                 }
4770             } else {
4771                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4772                 return;
4773             }
4774         } else {
4775             if ($force) {
4776                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4777                 kill 15, $pid;
4778             } else {
4779                 die "VM quit/powerdown failed\n";
4780             }
4781         }
4782
4783         # wait again
4784         $timeout = 10;
4785
4786         my $count = 0;
4787         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4788             $count++;
4789             sleep 1;
4790         }
4791
4792         if ($count >= $timeout) {
4793             warn "VM still running - terminating now with SIGKILL\n";
4794             kill 9, $pid;
4795             sleep 1;
4796         }
4797
4798         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4799    });
4800 }
4801
4802 sub vm_suspend {
4803     my ($vmid, $skiplock) = @_;
4804
4805     PVE::QemuConfig->lock_config($vmid, sub {
4806
4807         my $conf = PVE::QemuConfig->load_config($vmid);
4808
4809         PVE::QemuConfig->check_lock($conf)
4810             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4811
4812         vm_mon_cmd($vmid, "stop");
4813     });
4814 }
4815
4816 sub vm_resume {
4817     my ($vmid, $skiplock, $nocheck) = @_;
4818
4819     PVE::QemuConfig->lock_config($vmid, sub {
4820
4821         if (!$nocheck) {
4822
4823             my $conf = PVE::QemuConfig->load_config($vmid);
4824
4825             PVE::QemuConfig->check_lock($conf)
4826                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4827
4828             vm_mon_cmd($vmid, "cont");
4829
4830         } else {
4831             vm_mon_cmd_nocheck($vmid, "cont");
4832         }
4833     });
4834 }
4835
4836 sub vm_sendkey {
4837     my ($vmid, $skiplock, $key) = @_;
4838
4839     PVE::QemuConfig->lock_config($vmid, sub {
4840
4841         my $conf = PVE::QemuConfig->load_config($vmid);
4842
4843         # there is no qmp command, so we use the human monitor command
4844         vm_human_monitor_command($vmid, "sendkey $key");
4845     });
4846 }
4847
4848 sub vm_destroy {
4849     my ($storecfg, $vmid, $skiplock) = @_;
4850
4851     PVE::QemuConfig->lock_config($vmid, sub {
4852
4853         my $conf = PVE::QemuConfig->load_config($vmid);
4854
4855         if (!check_running($vmid)) {
4856             destroy_vm($storecfg, $vmid, undef, $skiplock);
4857         } else {
4858             die "VM $vmid is running - destroy failed\n";
4859         }
4860     });
4861 }
4862
4863 # pci helpers
4864
4865 sub file_write {
4866     my ($filename, $buf) = @_;
4867
4868     my $fh = IO::File->new($filename, "w");
4869     return undef if !$fh;
4870
4871     my $res = print $fh $buf;
4872
4873     $fh->close();
4874
4875     return $res;
4876 }
4877
4878 sub pci_device_info {
4879     my ($name) = @_;
4880
4881     my $res;
4882
4883     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4884     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4885
4886     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4887     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4888
4889     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4890     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4891
4892     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4893     return undef if !defined($product) || $product !~ s/^0x//;
4894
4895     $res = {
4896         name => $name,
4897         vendor => $vendor,
4898         product => $product,
4899         domain => $domain,
4900         bus => $bus,
4901         slot => $slot,
4902         func => $func,
4903         irq => $irq,
4904         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4905     };
4906
4907     return $res;
4908 }
4909
4910 sub pci_dev_reset {
4911     my ($dev) = @_;
4912
4913     my $name = $dev->{name};
4914
4915     my $fn = "$pcisysfs/devices/$name/reset";
4916
4917     return file_write($fn, "1");
4918 }
4919
4920 sub pci_dev_bind_to_vfio {
4921     my ($dev) = @_;
4922
4923     my $name = $dev->{name};
4924
4925     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4926
4927     if (!-d $vfio_basedir) {
4928         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4929     }
4930     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4931
4932     my $testdir = "$vfio_basedir/$name";
4933     return 1 if -d $testdir;
4934
4935     my $data = "$dev->{vendor} $dev->{product}";
4936     return undef if !file_write("$vfio_basedir/new_id", $data);
4937
4938     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4939     if (!file_write($fn, $name)) {
4940         return undef if -f $fn;
4941     }
4942
4943     $fn = "$vfio_basedir/bind";
4944     if (! -d $testdir) {
4945         return undef if !file_write($fn, $name);
4946     }
4947
4948     return -d $testdir;
4949 }
4950
4951 sub pci_dev_group_bind_to_vfio {
4952     my ($pciid) = @_;
4953
4954     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4955
4956     if (!-d $vfio_basedir) {
4957         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4958     }
4959     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4960
4961     # get IOMMU group devices
4962     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4963       my @devs = grep /^0000:/, readdir($D);
4964     closedir($D);
4965
4966     foreach my $pciid (@devs) {
4967         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4968
4969         # pci bridges, switches or root ports are not supported
4970         # they have a pci_bus subdirectory so skip them
4971         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4972
4973         my $info = pci_device_info($1);
4974         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4975     }
4976
4977     return 1;
4978 }
4979
4980 sub print_pci_addr {
4981     my ($id, $bridges) = @_;
4982
4983     my $res = '';
4984     my $devices = {
4985         piix3 => { bus => 0, addr => 1 },
4986         #addr2 : first videocard
4987         balloon0 => { bus => 0, addr => 3 },
4988         watchdog => { bus => 0, addr => 4 },
4989         scsihw0 => { bus => 0, addr => 5 },
4990         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4991         scsihw1 => { bus => 0, addr => 6 },
4992         ahci0 => { bus => 0, addr => 7 },
4993         qga0 => { bus => 0, addr => 8 },
4994         spice => { bus => 0, addr => 9 },
4995         virtio0 => { bus => 0, addr => 10 },
4996         virtio1 => { bus => 0, addr => 11 },
4997         virtio2 => { bus => 0, addr => 12 },
4998         virtio3 => { bus => 0, addr => 13 },
4999         virtio4 => { bus => 0, addr => 14 },
5000         virtio5 => { bus => 0, addr => 15 },
5001         hostpci0 => { bus => 0, addr => 16 },
5002         hostpci1 => { bus => 0, addr => 17 },
5003         net0 => { bus => 0, addr => 18 },
5004         net1 => { bus => 0, addr => 19 },
5005         net2 => { bus => 0, addr => 20 },
5006         net3 => { bus => 0, addr => 21 },
5007         net4 => { bus => 0, addr => 22 },
5008         net5 => { bus => 0, addr => 23 },
5009         vga1 => { bus => 0, addr => 24 },
5010         vga2 => { bus => 0, addr => 25 },
5011         vga3 => { bus => 0, addr => 26 },
5012         hostpci2 => { bus => 0, addr => 27 },
5013         hostpci3 => { bus => 0, addr => 28 },
5014         #addr29 : usb-host (pve-usb.cfg)
5015         'pci.1' => { bus => 0, addr => 30 },
5016         'pci.2' => { bus => 0, addr => 31 },
5017         'net6' => { bus => 1, addr => 1 },
5018         'net7' => { bus => 1, addr => 2 },
5019         'net8' => { bus => 1, addr => 3 },
5020         'net9' => { bus => 1, addr => 4 },
5021         'net10' => { bus => 1, addr => 5 },
5022         'net11' => { bus => 1, addr => 6 },
5023         'net12' => { bus => 1, addr => 7 },
5024         'net13' => { bus => 1, addr => 8 },
5025         'net14' => { bus => 1, addr => 9 },
5026         'net15' => { bus => 1, addr => 10 },
5027         'net16' => { bus => 1, addr => 11 },
5028         'net17' => { bus => 1, addr => 12 },
5029         'net18' => { bus => 1, addr => 13 },
5030         'net19' => { bus => 1, addr => 14 },
5031         'net20' => { bus => 1, addr => 15 },
5032         'net21' => { bus => 1, addr => 16 },
5033         'net22' => { bus => 1, addr => 17 },
5034         'net23' => { bus => 1, addr => 18 },
5035         'net24' => { bus => 1, addr => 19 },
5036         'net25' => { bus => 1, addr => 20 },
5037         'net26' => { bus => 1, addr => 21 },
5038         'net27' => { bus => 1, addr => 22 },
5039         'net28' => { bus => 1, addr => 23 },
5040         'net29' => { bus => 1, addr => 24 },
5041         'net30' => { bus => 1, addr => 25 },
5042         'net31' => { bus => 1, addr => 26 },
5043         'xhci' => { bus => 1, addr => 27 },
5044         'virtio6' => { bus => 2, addr => 1 },
5045         'virtio7' => { bus => 2, addr => 2 },
5046         'virtio8' => { bus => 2, addr => 3 },
5047         'virtio9' => { bus => 2, addr => 4 },
5048         'virtio10' => { bus => 2, addr => 5 },
5049         'virtio11' => { bus => 2, addr => 6 },
5050         'virtio12' => { bus => 2, addr => 7 },
5051         'virtio13' => { bus => 2, addr => 8 },
5052         'virtio14' => { bus => 2, addr => 9 },
5053         'virtio15' => { bus => 2, addr => 10 },
5054         'virtioscsi0' => { bus => 3, addr => 1 },
5055         'virtioscsi1' => { bus => 3, addr => 2 },
5056         'virtioscsi2' => { bus => 3, addr => 3 },
5057         'virtioscsi3' => { bus => 3, addr => 4 },
5058         'virtioscsi4' => { bus => 3, addr => 5 },
5059         'virtioscsi5' => { bus => 3, addr => 6 },
5060         'virtioscsi6' => { bus => 3, addr => 7 },
5061         'virtioscsi7' => { bus => 3, addr => 8 },
5062         'virtioscsi8' => { bus => 3, addr => 9 },
5063         'virtioscsi9' => { bus => 3, addr => 10 },
5064         'virtioscsi10' => { bus => 3, addr => 11 },
5065         'virtioscsi11' => { bus => 3, addr => 12 },
5066         'virtioscsi12' => { bus => 3, addr => 13 },
5067         'virtioscsi13' => { bus => 3, addr => 14 },
5068         'virtioscsi14' => { bus => 3, addr => 15 },
5069         'virtioscsi15' => { bus => 3, addr => 16 },
5070         'virtioscsi16' => { bus => 3, addr => 17 },
5071         'virtioscsi17' => { bus => 3, addr => 18 },
5072         'virtioscsi18' => { bus => 3, addr => 19 },
5073         'virtioscsi19' => { bus => 3, addr => 20 },
5074         'virtioscsi20' => { bus => 3, addr => 21 },
5075         'virtioscsi21' => { bus => 3, addr => 22 },
5076         'virtioscsi22' => { bus => 3, addr => 23 },
5077         'virtioscsi23' => { bus => 3, addr => 24 },
5078         'virtioscsi24' => { bus => 3, addr => 25 },
5079         'virtioscsi25' => { bus => 3, addr => 26 },
5080         'virtioscsi26' => { bus => 3, addr => 27 },
5081         'virtioscsi27' => { bus => 3, addr => 28 },
5082         'virtioscsi28' => { bus => 3, addr => 29 },
5083         'virtioscsi29' => { bus => 3, addr => 30 },
5084         'virtioscsi30' => { bus => 3, addr => 31 },
5085
5086     };
5087
5088     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5089            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5090            my $bus = $devices->{$id}->{bus};
5091            $res = ",bus=pci.$bus,addr=$addr";
5092            $bridges->{$bus} = 1 if $bridges;
5093     }
5094     return $res;
5095
5096 }
5097
5098 sub print_pcie_addr {
5099     my ($id) = @_;
5100
5101     my $res = '';
5102     my $devices = {
5103         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5104         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5105         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5106         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5107     };
5108
5109     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5110            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5111            my $bus = $devices->{$id}->{bus};
5112            $res = ",bus=$bus,addr=$addr";
5113     }
5114     return $res;
5115
5116 }
5117
5118 # vzdump restore implementaion
5119
5120 sub tar_archive_read_firstfile {
5121     my $archive = shift;
5122
5123     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5124
5125     # try to detect archive type first
5126     my $pid = open (TMP, "tar tf '$archive'|") ||
5127         die "unable to open file '$archive'\n";
5128     my $firstfile = <TMP>;
5129     kill 15, $pid;
5130     close TMP;
5131
5132     die "ERROR: archive contaions no data\n" if !$firstfile;
5133     chomp $firstfile;
5134
5135     return $firstfile;
5136 }
5137
5138 sub tar_restore_cleanup {
5139     my ($storecfg, $statfile) = @_;
5140
5141     print STDERR "starting cleanup\n";
5142
5143     if (my $fd = IO::File->new($statfile, "r")) {
5144         while (defined(my $line = <$fd>)) {
5145             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5146                 my $volid = $2;
5147                 eval {
5148                     if ($volid =~ m|^/|) {
5149                         unlink $volid || die 'unlink failed\n';
5150                     } else {
5151                         PVE::Storage::vdisk_free($storecfg, $volid);
5152                     }
5153                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5154                 };
5155                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5156             } else {
5157                 print STDERR "unable to parse line in statfile - $line";
5158             }
5159         }
5160         $fd->close();
5161     }
5162 }
5163
5164 sub restore_archive {
5165     my ($archive, $vmid, $user, $opts) = @_;
5166
5167     my $format = $opts->{format};
5168     my $comp;
5169
5170     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5171         $format = 'tar' if !$format;
5172         $comp = 'gzip';
5173     } elsif ($archive =~ m/\.tar$/) {
5174         $format = 'tar' if !$format;
5175     } elsif ($archive =~ m/.tar.lzo$/) {
5176         $format = 'tar' if !$format;
5177         $comp = 'lzop';
5178     } elsif ($archive =~ m/\.vma$/) {
5179         $format = 'vma' if !$format;
5180     } elsif ($archive =~ m/\.vma\.gz$/) {
5181         $format = 'vma' if !$format;
5182         $comp = 'gzip';
5183     } elsif ($archive =~ m/\.vma\.lzo$/) {
5184         $format = 'vma' if !$format;
5185         $comp = 'lzop';
5186     } else {
5187         $format = 'vma' if !$format; # default
5188     }
5189
5190     # try to detect archive format
5191     if ($format eq 'tar') {
5192         return restore_tar_archive($archive, $vmid, $user, $opts);
5193     } else {
5194         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5195     }
5196 }
5197
5198 sub restore_update_config_line {
5199     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5200
5201     return if $line =~ m/^\#qmdump\#/;
5202     return if $line =~ m/^\#vzdump\#/;
5203     return if $line =~ m/^lock:/;
5204     return if $line =~ m/^unused\d+:/;
5205     return if $line =~ m/^parent:/;
5206     return if $line =~ m/^template:/; # restored VM is never a template
5207
5208     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5209         # try to convert old 1.X settings
5210         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5211         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5212             my ($model, $macaddr) = split(/\=/, $devconfig);
5213             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5214             my $net = {
5215                 model => $model,
5216                 bridge => "vmbr$ind",
5217                 macaddr => $macaddr,
5218             };
5219             my $netstr = print_net($net);
5220
5221             print $outfd "net$cookie->{netcount}: $netstr\n";
5222             $cookie->{netcount}++;
5223         }
5224     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5225         my ($id, $netstr) = ($1, $2);
5226         my $net = parse_net($netstr);
5227         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5228         $netstr = print_net($net);
5229         print $outfd "$id: $netstr\n";
5230     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5231         my $virtdev = $1;
5232         my $value = $3;
5233         my $di = parse_drive($virtdev, $value);
5234         if (defined($di->{backup}) && !$di->{backup}) {
5235             print $outfd "#$line";
5236         } elsif ($map->{$virtdev}) {
5237             delete $di->{format}; # format can change on restore
5238             $di->{file} = $map->{$virtdev};
5239             $value = print_drive($vmid, $di);
5240             print $outfd "$virtdev: $value\n";
5241         } else {
5242             print $outfd $line;
5243         }
5244     } else {
5245         print $outfd $line;
5246     }
5247 }
5248
5249 sub scan_volids {
5250     my ($cfg, $vmid) = @_;
5251
5252     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5253
5254     my $volid_hash = {};
5255     foreach my $storeid (keys %$info) {
5256         foreach my $item (@{$info->{$storeid}}) {
5257             next if !($item->{volid} && $item->{size});
5258             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5259             $volid_hash->{$item->{volid}} = $item;
5260         }
5261     }
5262
5263     return $volid_hash;
5264 }
5265
5266 sub is_volume_in_use {
5267     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5268
5269     my $path = PVE::Storage::path($storecfg, $volid);
5270
5271     my $scan_config = sub {
5272         my ($cref, $snapname) = @_;
5273
5274         foreach my $key (keys %$cref) {
5275             my $value = $cref->{$key};
5276             if (is_valid_drivename($key)) {
5277                 next if $skip_drive && $key eq $skip_drive;
5278                 my $drive = parse_drive($key, $value);
5279                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5280                 return 1 if $volid eq $drive->{file};
5281                 if ($drive->{file} =~ m!^/!) {
5282                     return 1 if $drive->{file} eq $path;
5283                 } else {
5284                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5285                     next if !$storeid;
5286                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5287                     next if !$scfg;
5288                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5289                 }
5290             }
5291         }
5292
5293         return 0;
5294     };
5295
5296     return 1 if &$scan_config($conf);
5297
5298     undef $skip_drive;
5299
5300     foreach my $snapname (keys %{$conf->{snapshots}}) {
5301         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5302     }
5303
5304     return 0;
5305 }
5306
5307 sub update_disksize {
5308     my ($vmid, $conf, $volid_hash) = @_;
5309
5310     my $changes;
5311
5312     my $used = {};
5313
5314     # Note: it is allowed to define multiple storages with same path (alias), so
5315     # we need to check both 'volid' and real 'path' (two different volid can point
5316     # to the same path).
5317
5318     my $usedpath = {};
5319
5320     # update size info
5321     foreach my $opt (keys %$conf) {
5322         if (is_valid_drivename($opt)) {
5323             my $drive = parse_drive($opt, $conf->{$opt});
5324             my $volid = $drive->{file};
5325             next if !$volid;
5326
5327             $used->{$volid} = 1;
5328             if ($volid_hash->{$volid} &&
5329                 (my $path = $volid_hash->{$volid}->{path})) {
5330                 $usedpath->{$path} = 1;
5331             }
5332
5333             next if drive_is_cdrom($drive);
5334             next if !$volid_hash->{$volid};
5335
5336             $drive->{size} = $volid_hash->{$volid}->{size};
5337             my $new = print_drive($vmid, $drive);
5338             if ($new ne $conf->{$opt}) {
5339                 $changes = 1;
5340                 $conf->{$opt} = $new;
5341             }
5342         }
5343     }
5344
5345     # remove 'unusedX' entry if volume is used
5346     foreach my $opt (keys %$conf) {
5347         next if $opt !~ m/^unused\d+$/;
5348         my $volid = $conf->{$opt};
5349         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5350         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5351             $changes = 1;
5352             delete $conf->{$opt};
5353         }
5354     }
5355
5356     foreach my $volid (sort keys %$volid_hash) {
5357         next if $volid =~ m/vm-$vmid-state-/;
5358         next if $used->{$volid};
5359         my $path = $volid_hash->{$volid}->{path};
5360         next if !$path; # just to be sure
5361         next if $usedpath->{$path};
5362         $changes = 1;
5363         PVE::QemuConfig->add_unused_volume($conf, $volid);
5364         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5365     }
5366
5367     return $changes;
5368 }
5369
5370 sub rescan {
5371     my ($vmid, $nolock) = @_;
5372
5373     my $cfg = PVE::Storage::config();
5374
5375     my $volid_hash = scan_volids($cfg, $vmid);
5376
5377     my $updatefn =  sub {
5378         my ($vmid) = @_;
5379
5380         my $conf = PVE::QemuConfig->load_config($vmid);
5381
5382         PVE::QemuConfig->check_lock($conf);
5383
5384         my $vm_volids = {};
5385         foreach my $volid (keys %$volid_hash) {
5386             my $info = $volid_hash->{$volid};
5387             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5388         }
5389
5390         my $changes = update_disksize($vmid, $conf, $vm_volids);
5391
5392         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5393     };
5394
5395     if (defined($vmid)) {
5396         if ($nolock) {
5397             &$updatefn($vmid);
5398         } else {
5399             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5400         }
5401     } else {
5402         my $vmlist = config_list();
5403         foreach my $vmid (keys %$vmlist) {
5404             if ($nolock) {
5405                 &$updatefn($vmid);
5406             } else {
5407                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5408             }
5409         }
5410     }
5411 }
5412
5413 sub restore_vma_archive {
5414     my ($archive, $vmid, $user, $opts, $comp) = @_;
5415
5416     my $input = $archive eq '-' ? "<&STDIN" : undef;
5417     my $readfrom = $archive;
5418
5419     my $uncomp = '';
5420     if ($comp) {
5421         $readfrom = '-';
5422         my $qarchive = PVE::Tools::shellquote($archive);
5423         if ($comp eq 'gzip') {
5424             $uncomp = "zcat $qarchive|";
5425         } elsif ($comp eq 'lzop') {
5426             $uncomp = "lzop -d -c $qarchive|";
5427         } else {
5428             die "unknown compression method '$comp'\n";
5429         }
5430
5431     }
5432
5433     my $tmpdir = "/var/tmp/vzdumptmp$$";
5434     rmtree $tmpdir;
5435
5436     # disable interrupts (always do cleanups)
5437     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5438         warn "got interrupt - ignored\n";
5439     };
5440
5441     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5442     POSIX::mkfifo($mapfifo, 0600);
5443     my $fifofh;
5444
5445     my $openfifo = sub {
5446         open($fifofh, '>', $mapfifo) || die $!;
5447     };
5448
5449     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5450
5451     my $oldtimeout;
5452     my $timeout = 5;
5453
5454     my $devinfo = {};
5455
5456     my $rpcenv = PVE::RPCEnvironment::get();
5457
5458     my $conffile = PVE::QemuConfig->config_file($vmid);
5459     my $tmpfn = "$conffile.$$.tmp";
5460
5461     # Note: $oldconf is undef if VM does not exists
5462     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5463     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5464
5465     my $print_devmap = sub {
5466         my $virtdev_hash = {};
5467
5468         my $cfgfn = "$tmpdir/qemu-server.conf";
5469
5470         # we can read the config - that is already extracted
5471         my $fh = IO::File->new($cfgfn, "r") ||
5472             "unable to read qemu-server.conf - $!\n";
5473
5474         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5475         if (-f $fwcfgfn) {
5476             my $pve_firewall_dir = '/etc/pve/firewall';
5477             mkdir $pve_firewall_dir; # make sure the dir exists
5478             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5479         }
5480
5481         while (defined(my $line = <$fh>)) {
5482             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5483                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5484                 die "archive does not contain data for drive '$virtdev'\n"
5485                     if !$devinfo->{$devname};
5486                 if (defined($opts->{storage})) {
5487                     $storeid = $opts->{storage} || 'local';
5488                 } elsif (!$storeid) {
5489                     $storeid = 'local';
5490                 }
5491                 $format = 'raw' if !$format;
5492                 $devinfo->{$devname}->{devname} = $devname;
5493                 $devinfo->{$devname}->{virtdev} = $virtdev;
5494                 $devinfo->{$devname}->{format} = $format;
5495                 $devinfo->{$devname}->{storeid} = $storeid;
5496
5497                 # check permission on storage
5498                 my $pool = $opts->{pool}; # todo: do we need that?
5499                 if ($user ne 'root@pam') {
5500                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5501                 }
5502
5503                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5504             }
5505         }
5506
5507         foreach my $devname (keys %$devinfo) {
5508             die "found no device mapping information for device '$devname'\n"
5509                 if !$devinfo->{$devname}->{virtdev};
5510         }
5511
5512         my $cfg = PVE::Storage::config();
5513
5514         # create empty/temp config
5515         if ($oldconf) {
5516             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5517             foreach_drive($oldconf, sub {
5518                 my ($ds, $drive) = @_;
5519
5520                 return if drive_is_cdrom($drive);
5521
5522                 my $volid = $drive->{file};
5523
5524                 return if !$volid || $volid =~ m|^/|;
5525
5526                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5527                 return if !$path || !$owner || ($owner != $vmid);
5528
5529                 # Note: only delete disk we want to restore
5530                 # other volumes will become unused
5531                 if ($virtdev_hash->{$ds}) {
5532                     PVE::Storage::vdisk_free($cfg, $volid);
5533                 }
5534             });
5535
5536             # delete vmstate files
5537             # since after the restore we have no snapshots anymore
5538             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5539                 my $snap = $oldconf->{snapshots}->{$snapname};
5540                 if ($snap->{vmstate}) {
5541                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5542                     if (my $err = $@) {
5543                         warn $err;
5544                     }
5545                 }
5546             }
5547         }
5548
5549         my $map = {};
5550         foreach my $virtdev (sort keys %$virtdev_hash) {
5551             my $d = $virtdev_hash->{$virtdev};
5552             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5553             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5554
5555             # test if requested format is supported
5556             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5557             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5558             $d->{format} = $defFormat if !$supported;
5559
5560             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5561                                                   $d->{format}, undef, $alloc_size);
5562             print STDERR "new volume ID is '$volid'\n";
5563             $d->{volid} = $volid;
5564             my $path = PVE::Storage::path($cfg, $volid);
5565
5566             PVE::Storage::activate_volumes($cfg,[$volid]);
5567
5568             my $write_zeros = 1;
5569             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5570                 $write_zeros = 0;
5571             }
5572
5573             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5574
5575             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5576             $map->{$virtdev} = $volid;
5577         }
5578
5579         $fh->seek(0, 0) || die "seek failed - $!\n";
5580
5581         my $outfd = new IO::File ($tmpfn, "w") ||
5582             die "unable to write config for VM $vmid\n";
5583
5584         my $cookie = { netcount => 0 };
5585         while (defined(my $line = <$fh>)) {
5586             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5587         }
5588
5589         $fh->close();
5590         $outfd->close();
5591     };
5592
5593     eval {
5594         # enable interrupts
5595         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5596             die "interrupted by signal\n";
5597         };
5598         local $SIG{ALRM} = sub { die "got timeout\n"; };
5599
5600         $oldtimeout = alarm($timeout);
5601
5602         my $parser = sub {
5603             my $line = shift;
5604
5605             print "$line\n";
5606
5607             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5608                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5609                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5610             } elsif ($line =~ m/^CTIME: /) {
5611                 # we correctly received the vma config, so we can disable
5612                 # the timeout now for disk allocation (set to 10 minutes, so
5613                 # that we always timeout if something goes wrong)
5614                 alarm(600);
5615                 &$print_devmap();
5616                 print $fifofh "done\n";
5617                 my $tmp = $oldtimeout || 0;
5618                 $oldtimeout = undef;
5619                 alarm($tmp);
5620                 close($fifofh);
5621             }
5622         };
5623
5624         print "restore vma archive: $cmd\n";
5625         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5626     };
5627     my $err = $@;
5628
5629     alarm($oldtimeout) if $oldtimeout;
5630
5631     my $vollist = [];
5632     foreach my $devname (keys %$devinfo) {
5633         my $volid = $devinfo->{$devname}->{volid};
5634         push @$vollist, $volid if $volid;
5635     }
5636
5637     my $cfg = PVE::Storage::config();
5638     PVE::Storage::deactivate_volumes($cfg, $vollist);
5639
5640     unlink $mapfifo;
5641
5642     if ($err) {
5643         rmtree $tmpdir;
5644         unlink $tmpfn;
5645
5646         foreach my $devname (keys %$devinfo) {
5647             my $volid = $devinfo->{$devname}->{volid};
5648             next if !$volid;
5649             eval {
5650                 if ($volid =~ m|^/|) {
5651                     unlink $volid || die 'unlink failed\n';
5652                 } else {
5653                     PVE::Storage::vdisk_free($cfg, $volid);
5654                 }
5655                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5656             };
5657             print STDERR "unable to cleanup '$volid' - $@" if $@;
5658         }
5659         die $err;
5660     }
5661
5662     rmtree $tmpdir;
5663
5664     rename($tmpfn, $conffile) ||
5665         die "unable to commit configuration file '$conffile'\n";
5666
5667     PVE::Cluster::cfs_update(); # make sure we read new file
5668
5669     eval { rescan($vmid, 1); };
5670     warn $@ if $@;
5671 }
5672
5673 sub restore_tar_archive {
5674     my ($archive, $vmid, $user, $opts) = @_;
5675
5676     if ($archive ne '-') {
5677         my $firstfile = tar_archive_read_firstfile($archive);
5678         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5679             if $firstfile ne 'qemu-server.conf';
5680     }
5681
5682     my $storecfg = PVE::Storage::config();
5683
5684     # destroy existing data - keep empty config
5685     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5686     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5687
5688     my $tocmd = "/usr/lib/qemu-server/qmextract";
5689
5690     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5691     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5692     $tocmd .= ' --prealloc' if $opts->{prealloc};
5693     $tocmd .= ' --info' if $opts->{info};
5694
5695     # tar option "xf" does not autodetect compression when read from STDIN,
5696     # so we pipe to zcat
5697     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5698         PVE::Tools::shellquote("--to-command=$tocmd");
5699
5700     my $tmpdir = "/var/tmp/vzdumptmp$$";
5701     mkpath $tmpdir;
5702
5703     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5704     local $ENV{VZDUMP_VMID} = $vmid;
5705     local $ENV{VZDUMP_USER} = $user;
5706
5707     my $conffile = PVE::QemuConfig->config_file($vmid);
5708     my $tmpfn = "$conffile.$$.tmp";
5709
5710     # disable interrupts (always do cleanups)
5711     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5712         print STDERR "got interrupt - ignored\n";
5713     };
5714
5715     eval {
5716         # enable interrupts
5717         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5718             die "interrupted by signal\n";
5719         };
5720
5721         if ($archive eq '-') {
5722             print "extracting archive from STDIN\n";
5723             run_command($cmd, input => "<&STDIN");
5724         } else {
5725             print "extracting archive '$archive'\n";
5726             run_command($cmd);
5727         }
5728
5729         return if $opts->{info};
5730
5731         # read new mapping
5732         my $map = {};
5733         my $statfile = "$tmpdir/qmrestore.stat";
5734         if (my $fd = IO::File->new($statfile, "r")) {
5735             while (defined (my $line = <$fd>)) {
5736                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5737                     $map->{$1} = $2 if $1;
5738                 } else {
5739                     print STDERR "unable to parse line in statfile - $line\n";
5740                 }
5741             }
5742             $fd->close();
5743         }
5744
5745         my $confsrc = "$tmpdir/qemu-server.conf";
5746
5747         my $srcfd = new IO::File($confsrc, "r") ||
5748             die "unable to open file '$confsrc'\n";
5749
5750         my $outfd = new IO::File ($tmpfn, "w") ||
5751             die "unable to write config for VM $vmid\n";
5752
5753         my $cookie = { netcount => 0 };
5754         while (defined (my $line = <$srcfd>)) {
5755             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5756         }
5757
5758         $srcfd->close();
5759         $outfd->close();
5760     };
5761     my $err = $@;
5762
5763     if ($err) {
5764
5765         unlink $tmpfn;
5766
5767         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5768
5769         die $err;
5770     }
5771
5772     rmtree $tmpdir;
5773
5774     rename $tmpfn, $conffile ||
5775         die "unable to commit configuration file '$conffile'\n";
5776
5777     PVE::Cluster::cfs_update(); # make sure we read new file
5778
5779     eval { rescan($vmid, 1); };
5780     warn $@ if $@;
5781 };
5782
5783 sub foreach_writable_storage {
5784     my ($conf, $func) = @_;
5785
5786     my $sidhash = {};
5787
5788     foreach my $ds (keys %$conf) {
5789         next if !is_valid_drivename($ds);
5790
5791         my $drive = parse_drive($ds, $conf->{$ds});
5792         next if !$drive;
5793         next if drive_is_cdrom($drive);
5794
5795         my $volid = $drive->{file};
5796
5797         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5798         $sidhash->{$sid} = $sid if $sid;
5799     }
5800
5801     foreach my $sid (sort keys %$sidhash) {
5802         &$func($sid);
5803     }
5804 }
5805
5806 sub do_snapshots_with_qemu {
5807     my ($storecfg, $volid) = @_;
5808
5809     my $storage_name = PVE::Storage::parse_volume_id($volid);
5810
5811     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5812         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5813         return 1;
5814     }
5815
5816     if ($volid =~ m/\.(qcow2|qed)$/){
5817         return 1;
5818     }
5819
5820     return undef;
5821 }
5822
5823 sub qga_check_running {
5824     my ($vmid) = @_;
5825
5826     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5827     if ($@) {
5828         warn "Qemu Guest Agent are not running - $@";
5829         return 0;
5830     }
5831     return 1;
5832 }
5833
5834 sub template_create {
5835     my ($vmid, $conf, $disk) = @_;
5836
5837     my $storecfg = PVE::Storage::config();
5838
5839     foreach_drive($conf, sub {
5840         my ($ds, $drive) = @_;
5841
5842         return if drive_is_cdrom($drive);
5843         return if $disk && $ds ne $disk;
5844
5845         my $volid = $drive->{file};
5846         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5847
5848         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5849         $drive->{file} = $voliddst;
5850         $conf->{$ds} = print_drive($vmid, $drive);
5851         PVE::QemuConfig->write_config($vmid, $conf);
5852     });
5853 }
5854
5855 sub qemu_img_convert {
5856     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5857
5858     my $storecfg = PVE::Storage::config();
5859     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5860     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5861
5862     if ($src_storeid && $dst_storeid) {
5863
5864         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5865
5866         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5867         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5868
5869         my $src_format = qemu_img_format($src_scfg, $src_volname);
5870         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5871
5872         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5873         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5874
5875         my $cmd = [];
5876         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5877         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5878         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5879         if ($is_zero_initialized) {
5880             push @$cmd, "zeroinit:$dst_path";
5881         } else {
5882             push @$cmd, $dst_path;
5883         }
5884
5885         my $parser = sub {
5886             my $line = shift;
5887             if($line =~ m/\((\S+)\/100\%\)/){
5888                 my $percent = $1;
5889                 my $transferred = int($size * $percent / 100);
5890                 my $remaining = $size - $transferred;
5891
5892                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5893             }
5894
5895         };
5896
5897         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5898         my $err = $@;
5899         die "copy failed: $err" if $err;
5900     }
5901 }
5902
5903 sub qemu_img_format {
5904     my ($scfg, $volname) = @_;
5905
5906     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5907         return $1;
5908     } else {
5909         return "raw";
5910     }
5911 }
5912
5913 sub qemu_drive_mirror {
5914     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5915
5916     my $storecfg = PVE::Storage::config();
5917     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5918
5919     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5920
5921     my $format = qemu_img_format($dst_scfg, $dst_volname);
5922
5923     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5924
5925     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5926
5927     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5928     $opts->{format} = $format if $format;
5929
5930     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5931
5932     my $finish_job = sub {
5933         while (1) {
5934             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5935             my $stat = @$stats[0];
5936             last if !$stat;
5937             sleep 1;
5938         }
5939     };
5940
5941     eval {
5942     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5943         while (1) {
5944             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5945             my $stat = @$stats[0];
5946             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5947             die "error job is not mirroring" if $stat->{type} ne "mirror";
5948
5949             my $busy = $stat->{busy};
5950             my $ready = $stat->{ready};
5951
5952             if (my $total = $stat->{len}) {
5953                 my $transferred = $stat->{offset} || 0;
5954                 my $remaining = $total - $transferred;
5955                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5956
5957                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5958             }
5959
5960
5961             if ($stat->{ready} eq 'true') {
5962
5963                 last if $vmiddst != $vmid;
5964
5965                 # try to switch the disk if source and destination are on the same guest
5966                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5967                 if (!$@) {
5968                     &$finish_job();
5969                     last;
5970                 }
5971                 die $@ if $@ !~ m/cannot be completed/;
5972             }
5973             sleep 1;
5974         }
5975
5976
5977     };
5978     my $err = $@;
5979
5980     my $cancel_job = sub {
5981         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5982         &$finish_job();
5983     };
5984
5985     if ($err) {
5986         eval { &$cancel_job(); };
5987         die "mirroring error: $err";
5988     }
5989
5990     if ($vmiddst != $vmid) {
5991         # if we clone a disk for a new target vm, we don't switch the disk
5992         &$cancel_job(); # so we call block-job-cancel
5993     }
5994 }
5995
5996 sub clone_disk {
5997     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5998         $newvmid, $storage, $format, $full, $newvollist) = @_;
5999
6000     my $newvolid;
6001
6002     if (!$full) {
6003         print "create linked clone of drive $drivename ($drive->{file})\n";
6004         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6005         push @$newvollist, $newvolid;
6006     } else {
6007         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6008         $storeid = $storage if $storage;
6009
6010         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6011         if (!$format) {
6012             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6013             $format = qemu_img_format($scfg, $volname);
6014         }
6015
6016         # test if requested format is supported - else use default
6017         my $supported = grep { $_ eq $format } @$validFormats;
6018         $format = $defFormat if !$supported;
6019
6020         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6021
6022         print "create full clone of drive $drivename ($drive->{file})\n";
6023         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6024         push @$newvollist, $newvolid;
6025
6026         PVE::Storage::activate_volumes($storecfg, $newvollist);
6027
6028         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6029         if (!$running || $snapname) {
6030             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6031         } else {
6032             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
6033         }
6034     }
6035
6036     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6037
6038     my $disk = $drive;
6039     $disk->{format} = undef;
6040     $disk->{file} = $newvolid;
6041     $disk->{size} = $size;
6042
6043     return $disk;
6044 }
6045
6046 # this only works if VM is running
6047 sub get_current_qemu_machine {
6048     my ($vmid) = @_;
6049
6050     my $cmd = { execute => 'query-machines', arguments => {} };
6051     my $res = vm_qmp_command($vmid, $cmd);
6052
6053     my ($current, $default);
6054     foreach my $e (@$res) {
6055         $default = $e->{name} if $e->{'is-default'};
6056         $current = $e->{name} if $e->{'is-current'};
6057     }
6058
6059     # fallback to the default machine if current is not supported by qemu
6060     return $current || $default || 'pc';
6061 }
6062
6063 sub qemu_machine_feature_enabled {
6064     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6065
6066     my $current_major;
6067     my $current_minor;
6068
6069     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6070
6071         $current_major = $3;
6072         $current_minor = $4;
6073
6074     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6075
6076         $current_major = $1;
6077         $current_minor = $2;
6078     }
6079
6080     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6081
6082
6083 }
6084
6085 sub qemu_machine_pxe {
6086     my ($vmid, $conf, $machine) = @_;
6087
6088     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6089
6090     foreach my $opt (keys %$conf) {
6091         next if $opt !~ m/^net(\d+)$/;
6092         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6093         next if !$net;
6094         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6095         return $machine.".pxe" if $romfile =~ m/pxe/;
6096         last;
6097     }
6098
6099     return $machine;
6100 }
6101
6102 sub qemu_use_old_bios_files {
6103     my ($machine_type) = @_;
6104
6105     return if !$machine_type;
6106
6107     my $use_old_bios_files = undef;
6108
6109     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6110         $machine_type = $1;
6111         $use_old_bios_files = 1;
6112     } else {
6113         my $kvmver = kvm_user_version();
6114         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6115         # load new efi bios files on migration. So this hack is required to allow
6116         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6117         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6118         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6119     }
6120
6121     return ($use_old_bios_files, $machine_type);
6122 }
6123
6124 sub lspci {
6125
6126     my $devices = {};
6127
6128     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6129             my (undef, $id, $function) = @_;
6130             my $res = { id => $id, function => $function};
6131             push @{$devices->{$id}}, $res;
6132     });
6133
6134     return $devices;
6135 }
6136
6137 sub vm_iothreads_list {
6138     my ($vmid) = @_;
6139
6140     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6141
6142     my $iothreads = {};
6143     foreach my $iothread (@$res) {
6144         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6145     }
6146
6147     return $iothreads;
6148 }
6149
6150 sub scsihw_infos {
6151     my ($conf, $drive) = @_;
6152
6153     my $maxdev = 0;
6154
6155     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6156         $maxdev = 7;
6157     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6158         $maxdev = 1;
6159     } else {
6160         $maxdev = 256;
6161     }
6162
6163     my $controller = int($drive->{index} / $maxdev);
6164     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6165
6166     return ($maxdev, $controller, $controller_prefix);
6167 }
6168
6169 # bash completion helper
6170
6171 sub complete_backup_archives {
6172     my ($cmdname, $pname, $cvalue) = @_;
6173
6174     my $cfg = PVE::Storage::config();
6175
6176     my $storeid;
6177
6178     if ($cvalue =~ m/^([^:]+):/) {
6179         $storeid = $1;
6180     }
6181
6182     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6183
6184     my $res = [];
6185     foreach my $id (keys %$data) {
6186         foreach my $item (@{$data->{$id}}) {
6187             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6188             push @$res, $item->{volid} if defined($item->{volid});
6189         }
6190     }
6191
6192     return $res;
6193 }
6194
6195 my $complete_vmid_full = sub {
6196     my ($running) = @_;
6197
6198     my $idlist = vmstatus();
6199
6200     my $res = [];
6201
6202     foreach my $id (keys %$idlist) {
6203         my $d = $idlist->{$id};
6204         if (defined($running)) {
6205             next if $d->{template};
6206             next if $running && $d->{status} ne 'running';
6207             next if !$running && $d->{status} eq 'running';
6208         }
6209         push @$res, $id;
6210
6211     }
6212     return $res;
6213 };
6214
6215 sub complete_vmid {
6216     return &$complete_vmid_full();
6217 }
6218
6219 sub complete_vmid_stopped {
6220     return &$complete_vmid_full(0);
6221 }
6222
6223 sub complete_vmid_running {
6224     return &$complete_vmid_full(1);
6225 }
6226
6227 sub complete_storage {
6228
6229     my $cfg = PVE::Storage::config();
6230     my $ids = $cfg->{ids};
6231
6232     my $res = [];
6233     foreach my $sid (keys %$ids) {
6234         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6235         next if !$ids->{$sid}->{content}->{images};
6236         push @$res, $sid;
6237     }
6238
6239     return $res;
6240 }
6241
6242 1;