PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35 use URI::Escape;
  36
  37 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  38
  39 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  40
  41 # Note about locking: we use flock on the config file protect
  42 # against concurent actions.
  43 # Aditionaly, we have a 'lock' setting in the config file. This
  44 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  45 # allowed when such lock is set. But you can ignore this kind of
  46 # lock with the --skiplock flag.
  47
  48 cfs_register_file('/qemu-server/',
  49                   \&parse_vm_config,
  50                   \&write_vm_config);
  51
  52 PVE::JSONSchema::register_standard_option('skiplock', {
  53     description => "Ignore locks - only root is allowed to use this option.",
  54     type => 'boolean',
  55     optional => 1,
  56 });
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 #no warnings 'redefine';
  72
  73 sub cgroups_write {
  74    my ($controller, $vmid, $option, $value) = @_;
  75
  76    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  77    PVE::ProcFSTools::write_proc_entry($path, $value);
  78
  79 }
  80
  81 my $nodename = PVE::INotify::nodename();
  82
  83 mkdir "/etc/pve/nodes/$nodename";
  84 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  85 mkdir $confdir;
  86
  87 my $var_run_tmpdir = "/var/run/qemu-server";
  88 mkdir $var_run_tmpdir;
  89
  90 my $lock_dir = "/var/lock/qemu-server";
  91 mkdir $lock_dir;
  92
  93 my $pcisysfs = "/sys/bus/pci";
  94
  95 my $cpu_fmt = {
  96     cputype => {
  97         description => "Emulated CPU type.",
  98         type => 'string',
  99         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 100         format_description => 'cputype',
 101         default => 'kvm64',
 102         default_key => 1,
 103     },
 104     hidden => {
 105         description => "Do not identify as a KVM virtual machine.",
 106         type => 'boolean',
 107         optional => 1,
 108         default => 0
 109     },
 110 };
 111
 112 my $watchdog_fmt = {
 113     model => {
 114         default_key => 1,
 115         type => 'string',
 116         enum => [qw(i6300esb ib700)],
 117         description => "Watchdog type to emulate.",
 118         default => 'i6300esb',
 119         optional => 1,
 120     },
 121     action => {
 122         type => 'string',
 123         enum => [qw(reset shutdown poweroff pause debug none)],
 124         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 125         optional => 1,
 126     },
 127 };
 128 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 129
 130 my $confdesc = {
 131     onboot => {
 132         optional => 1,
 133         type => 'boolean',
 134         description => "Specifies whether a VM will be started during system bootup.",
 135         default => 0,
 136     },
 137     autostart => {
 138         optional => 1,
 139         type => 'boolean',
 140         description => "Automatic restart after crash (currently ignored).",
 141         default => 0,
 142     },
 143     hotplug => {
 144         optional => 1,
 145         type => 'string', format => 'pve-hotplug-features',
 146         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 147         default => 'network,disk,usb',
 148     },
 149     reboot => {
 150         optional => 1,
 151         type => 'boolean',
 152         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 153         default => 1,
 154     },
 155     lock => {
 156         optional => 1,
 157         type => 'string',
 158         description => "Lock/unlock the VM.",
 159         enum => [qw(migrate backup snapshot rollback)],
 160     },
 161     cpulimit => {
 162         optional => 1,
 163         type => 'number',
 164         description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 165         minimum => 0,
 166         maximum => 128,
 167         default => 0,
 168     },
 169     cpuunits => {
 170         optional => 1,
 171         type => 'integer',
 172         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 173         minimum => 0,
 174         maximum => 500000,
 175         default => 1000,
 176     },
 177     memory => {
 178         optional => 1,
 179         type => 'integer',
 180         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 181         minimum => 16,
 182         default => 512,
 183     },
 184     balloon => {
 185         optional => 1,
 186         type => 'integer',
 187         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 188         minimum => 0,
 189     },
 190     shares => {
 191         optional => 1,
 192         type => 'integer',
 193         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 194         minimum => 0,
 195         maximum => 50000,
 196         default => 1000,
 197     },
 198     keyboard => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 202         enum => PVE::Tools::kvmkeymaplist(),
 203         default => 'en-us',
 204     },
 205     name => {
 206         optional => 1,
 207         type => 'string', format => 'dns-name',
 208         description => "Set a name for the VM. Only used on the configuration web interface.",
 209     },
 210     scsihw => {
 211         optional => 1,
 212         type => 'string',
 213         description => "scsi controller model",
 214         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 215         default => 'lsi',
 216     },
 217     description => {
 218         optional => 1,
 219         type => 'string',
 220         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 221     },
 222     ostype => {
 223         optional => 1,
 224         type => 'string',
 225         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 226         description => <<EODESC,
 227 Used to enable special optimization/features for specific
 228 operating systems:
 229
 230 other  => unspecified OS
 231 wxp    => Microsoft Windows XP
 232 w2k    => Microsoft Windows 2000
 233 w2k3   => Microsoft Windows 2003
 234 w2k8   => Microsoft Windows 2008
 235 wvista => Microsoft Windows Vista
 236 win7   => Microsoft Windows 7
 237 win8   => Microsoft Windows 8/2012
 238 l24    => Linux 2.4 Kernel
 239 l26    => Linux 2.6/3.X Kernel
 240 solaris => solaris/opensolaris/openindiania kernel
 241
 242 other|l24|l26|solaris                       ... no special behaviour
 243 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 244 EODESC
 245     },
 246     boot => {
 247         optional => 1,
 248         type => 'string',
 249         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 250         pattern => '[acdn]{1,4}',
 251         default => 'cdn',
 252     },
 253     bootdisk => {
 254         optional => 1,
 255         type => 'string', format => 'pve-qm-bootdisk',
 256         description => "Enable booting from specified disk.",
 257         pattern => '(ide|sata|scsi|virtio)\d+',
 258     },
 259     smp => {
 260         optional => 1,
 261         type => 'integer',
 262         description => "The number of CPUs. Please use option -sockets instead.",
 263         minimum => 1,
 264         default => 1,
 265     },
 266     sockets => {
 267         optional => 1,
 268         type => 'integer',
 269         description => "The number of CPU sockets.",
 270         minimum => 1,
 271         default => 1,
 272     },
 273     cores => {
 274         optional => 1,
 275         type => 'integer',
 276         description => "The number of cores per socket.",
 277         minimum => 1,
 278         default => 1,
 279     },
 280     numa => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Enable/disable NUMA.",
 284         default => 0,
 285     },
 286     vcpus => {
 287         optional => 1,
 288         type => 'integer',
 289         description => "Number of hotplugged vcpus.",
 290         minimum => 1,
 291         default => 0,
 292     },
 293     acpi => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Enable/disable ACPI.",
 297         default => 1,
 298     },
 299     agent => {
 300         optional => 1,
 301         type => 'boolean',
 302         description => "Enable/disable Qemu GuestAgent.",
 303         default => 0,
 304     },
 305     kvm => {
 306         optional => 1,
 307         type => 'boolean',
 308         description => "Enable/disable KVM hardware virtualization.",
 309         default => 1,
 310     },
 311     tdf => {
 312         optional => 1,
 313         type => 'boolean',
 314         description => "Enable/disable time drift fix.",
 315         default => 0,
 316     },
 317     localtime => {
 318         optional => 1,
 319         type => 'boolean',
 320         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 321     },
 322     freeze => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 326     },
 327     vga => {
 328         optional => 1,
 329         type => 'string',
 330         description => "Select the VGA type. If you want to use high resolution" .
 331             " modes (>= 1280x1024x16) then you should use the options " .
 332             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 333             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 334             "display sever. For win* OS you can select how many independent " .
 335             "displays you want, Linux guests can add displays them self. " .
 336             "You can also run without any graphic card, using a serial device" .
 337             " as terminal.",
 338         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 339     },
 340     watchdog => {
 341         optional => 1,
 342         type => 'string', format => 'pve-qm-watchdog',
 343         description => "Create a virtual hardware watchdog device. Once enabled" .
 344             " (by a guest action), the watchdog must be periodically polled " .
 345             "by an agent inside the guest or else the watchdog will reset " .
 346             "the guest (or execute the respective action specified)",
 347     },
 348     startdate => {
 349         optional => 1,
 350         type => 'string',
 351         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 352         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 353         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 354         default => 'now',
 355     },
 356     startup =>  get_standard_option('pve-startup-order'),
 357     template => {
 358         optional => 1,
 359         type => 'boolean',
 360         description => "Enable/disable Template.",
 361         default => 0,
 362     },
 363     args => {
 364         optional => 1,
 365         type => 'string',
 366         description => <<EODESCR,
 367 Arbitrary arguments passed to kvm, for example:
 368
 369 args: -no-reboot -no-hpet
 370
 371 NOTE: this option is for experts only.
 372 EODESCR
 373     },
 374     tablet => {
 375         optional => 1,
 376         type => 'boolean',
 377         default => 1,
 378         description => "Enable/disable the USB tablet device. This device is " .
 379             "usually needed to allow absolute mouse positioning with VNC. " .
 380             "Else the mouse runs out of sync with normal VNC clients. " .
 381             "If you're running lots of console-only guests on one host, " .
 382             "you may consider disabling this to save some context switches. " .
 383             "This is turned off by default if you use spice (-vga=qxl).",
 384     },
 385     migrate_speed => {
 386         optional => 1,
 387         type => 'integer',
 388         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 389         minimum => 0,
 390         default => 0,
 391     },
 392     migrate_downtime => {
 393         optional => 1,
 394         type => 'number',
 395         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 396         minimum => 0,
 397         default => 0.1,
 398     },
 399     cdrom => {
 400         optional => 1,
 401         type => 'string', format => 'pve-qm-drive',
 402         typetext => 'volume',
 403         description => "This is an alias for option -ide2",
 404     },
 405     cpu => {
 406         optional => 1,
 407         description => "Emulated CPU type.",
 408         type => 'string',
 409         format => $cpu_fmt,
 410     },
 411     parent => get_standard_option('pve-snapshot-name', {
 412         optional => 1,
 413         description => "Parent snapshot name. This is used internally, and should not be modified.",
 414     }),
 415     snaptime => {
 416         optional => 1,
 417         description => "Timestamp for snapshots.",
 418         type => 'integer',
 419         minimum => 0,
 420     },
 421     vmstate => {
 422         optional => 1,
 423         type => 'string', format => 'pve-volume-id',
 424         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 425     },
 426     machine => {
 427         description => "Specific the Qemu machine type.",
 428         type => 'string',
 429         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 430         maxLength => 40,
 431         optional => 1,
 432     },
 433     smbios1 => {
 434         description => "Specify SMBIOS type 1 fields.",
 435         type => 'string', format => 'pve-qm-smbios1',
 436         maxLength => 256,
 437         optional => 1,
 438     },
 439     protection => {
 440         optional => 1,
 441         type => 'boolean',
 442         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 443         default => 0,
 444     },
 445     bios => {
 446         optional => 1,
 447         type => 'string',
 448         enum => [ qw(seabios ovmf) ],
 449         description => "Select BIOS implementation.",
 450         default => 'seabios',
 451     },
 452 };
 453
 454 # what about other qemu settings ?
 455 #cpu => 'string',
 456 #machine => 'string',
 457 #fda => 'file',
 458 #fdb => 'file',
 459 #mtdblock => 'file',
 460 #sd => 'file',
 461 #pflash => 'file',
 462 #snapshot => 'bool',
 463 #bootp => 'file',
 464 ##tftp => 'dir',
 465 ##smb => 'dir',
 466 #kernel => 'file',
 467 #append => 'string',
 468 #initrd => 'file',
 469 ##soundhw => 'string',
 470
 471 while (my ($k, $v) = each %$confdesc) {
 472     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 473 }
 474
 475 my $MAX_IDE_DISKS = 4;
 476 my $MAX_SCSI_DISKS = 14;
 477 my $MAX_VIRTIO_DISKS = 16;
 478 my $MAX_SATA_DISKS = 6;
 479 my $MAX_USB_DEVICES = 5;
 480 my $MAX_NETS = 32;
 481 my $MAX_UNUSED_DISKS = 8;
 482 my $MAX_HOSTPCI_DEVICES = 4;
 483 my $MAX_SERIAL_PORTS = 4;
 484 my $MAX_PARALLEL_PORTS = 3;
 485 my $MAX_NUMA = 8;
 486 my $MAX_MEM = 4194304;
 487 my $STATICMEM = 1024;
 488
 489 my $numa_fmt = {
 490     cpus => {
 491         type => "string",
 492         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 493         description => "CPUs accessing this numa node.",
 494         format_description => "id[-id];...",
 495     },
 496     memory => {
 497         type => "number",
 498         description => "Amount of memory this numa node provides.",
 499         format_description => "mb",
 500         optional => 1,
 501     },
 502     hostnodes => {
 503         type => "string",
 504         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 505         description => "host numa nodes to use",
 506         format_description => "id[-id];...",
 507         optional => 1,
 508     },
 509     policy => {
 510         type => 'string',
 511         enum => [qw(preferred bind interleave)],
 512         format_description => 'preferred|bind|interleave',
 513         description => "numa allocation policy.",
 514         optional => 1,
 515     },
 516 };
 517 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 518 my $numadesc = {
 519     optional => 1,
 520     type => 'string', format => $numa_fmt,
 521     description => "numa topology",
 522 };
 523 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 524
 525 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 526     $confdesc->{"numa$i"} = $numadesc;
 527 }
 528
 529 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 530                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 531                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 532 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 533
 534 my $net_fmt = {
 535     macaddr => {
 536         type => 'string',
 537         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 538         description => "MAC address",
 539         format_description => "XX:XX:XX:XX:XX:XX",
 540         optional => 1,
 541     },
 542     model => { alias => 'macaddr', default_key => 1 },
 543     (map { $_ => { group => 'model' } } @$nic_model_list),
 544     bridge => {
 545         type => 'string',
 546         description => 'Bridge to attach the network device to.',
 547         format_description => 'bridge',
 548         optional => 1,
 549     },
 550     queues => {
 551         type => 'integer',
 552         minimum => 0, maximum => 16,
 553         description => 'Number of packet queues to be used on the device.',
 554         format_description => 'number',
 555         optional => 1,
 556     },
 557     rate => {
 558         type => 'number',
 559         minimum => 0,
 560         description => 'Rate limit in mbps as floating point number.',
 561         format_description => 'mbps',
 562         optional => 1,
 563     },
 564     tag => {
 565         type => 'integer',
 566         minimum => 2, maximum => 4094,
 567         description => 'VLAN tag to apply to packets on this interface.',
 568         format_description => 'vlanid',
 569         optional => 1,
 570     },
 571     trunks => {
 572         type => 'string',
 573         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 574         description => 'VLAN trunks to pass through this interface.',
 575         format_description => 'id;id...',
 576         optional => 1,
 577     },
 578     firewall => {
 579         type => 'boolean',
 580         description => 'Whether this interface should be protected by the firewall.',
 581         format_description => '0|1',
 582         optional => 1,
 583     },
 584     link_down => {
 585         type => 'boolean',
 586         description => 'Whether this interface should be DISconnected (like pulling the plug).',
 587         format_description => '0|1',
 588         optional => 1,
 589     },
 590 };
 591 my $netdesc = {
 592     optional => 1,
 593     type => 'string', format => 'pve-qm-net',
 594     description => <<EODESCR,
 595 Specify network devices.
 596
 597 MODEL is one of: $nic_model_list_txt
 598
 599 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 600 automatically generated if not specified.
 601
 602 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 603
 604 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 605
 606 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 607
 608 10.0.2.2   Gateway
 609 10.0.2.3   DNS Server
 610 10.0.2.4   SMB Server
 611
 612 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 613
 614 EODESCR
 615 };
 616 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 617
 618 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 619     $confdesc->{"net$i"} = $netdesc;
 620 }
 621
 622 PVE::JSONSchema::register_format('pve-volume-id-or-none', \&verify_volume_id_or_none);
 623 sub verify_volume_id_or_none {
 624     my ($volid, $noerr) = @_;
 625
 626     return $volid if $volid eq 'none';
 627     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 628     if ($@) {
 629         return undef if $noerr;
 630         die $@;
 631     }
 632     return $volid;
 633 }
 634
 635 my $drivename_hash;
 636
 637 my %drivedesc_base = (
 638     volume => { alias => 'file' },
 639     file => {
 640         type => 'string',
 641         format => 'pve-volume-id-or-none',
 642         default_key => 1,
 643         format_description => 'volume',
 644         description => "The drive's backing volume.",
 645     },
 646     media => {
 647         type => 'string',
 648         format_description => 'cdrom|disk',
 649         enum => [qw(cdrom disk)],
 650         description => "The drive's media type.",
 651         default => 'disk',
 652         optional => 1
 653     },
 654     cyls => {
 655         type => 'integer',
 656         format_description => 'count',
 657         description => "Force the drive's physical geometry to have a specific cylinder count.",
 658         optional => 1
 659     },
 660     heads => {
 661         type => 'integer',
 662         format_description => 'count',
 663         description => "Force the drive's physical geometry to have a specific head count.",
 664         optional => 1
 665     },
 666     secs => {
 667         type => 'integer',
 668         format_description => 'count',
 669         description => "Force the drive's physical geometry to have a specific sector count.",
 670         optional => 1
 671     },
 672     trans => {
 673         type => 'string',
 674         format_description => 'none|lba|auto',
 675         enum => [qw(none lba auto)],
 676         description => "Force disk geometry bios translation mode.",
 677         optional => 1,
 678     },
 679     snapshot => {
 680         type => 'boolean',
 681         format_description => 'on|off',
 682         description => "Whether the drive should be included when making snapshots.",
 683         optional => 1,
 684     },
 685     cache => {
 686         type => 'string',
 687         format_description => 'none|writethrough|writeback|unsafe|directsync',
 688         enum => [qw(none writethrough writeback unsafe directsync)],
 689         description => "The drive's cache mode",
 690         optional => 1,
 691     },
 692     format => {
 693         type => 'string',
 694         format_description => 'drive format',
 695         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 696         description => "The drive's backing file's data format.",
 697         optional => 1,
 698     },
 699     size => {
 700         type => 'string',
 701         format => 'disk-size',
 702         description => "Disk size. This is purely informational and has no effect.",
 703         optional => 1,
 704     },
 705     backup => {
 706         type => 'boolean',
 707         format_description => 'on|off',
 708         description => "Whether the drive should be included when making backups.",
 709         optional => 1,
 710     },
 711     werror => {
 712         type => 'string',
 713         format_description => 'enospc|ignore|report|stop',
 714         enum => [qw(enospc ignore report stop)],
 715         description => 'Write error action.',
 716         optional => 1,
 717     },
 718     aio => {
 719         type => 'string',
 720         format_description => 'native|threads',
 721         enum => [qw(native threads)],
 722         description => 'AIO type to use.',
 723         optional => 1,
 724     },
 725     discard => {
 726         type => 'string',
 727         format_description => 'ignore|on',
 728         enum => [qw(ignore on)],
 729         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 730         optional => 1,
 731     },
 732     detect_zeroes => {
 733         type => 'boolean',
 734         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 735         optional => 1,
 736     },
 737     serial => {
 738         type => 'string',
 739         format => 'urlencoded',
 740         format_description => 'serial',
 741         maxLength => 20*3, # *3 since it's %xx url enoded
 742         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 743         optional => 1,
 744     }
 745 );
 746
 747 my %rerror_fmt = (
 748     rerror => {
 749         type => 'string',
 750         format_description => 'ignore|report|stop',
 751         enum => [qw(ignore report stop)],
 752         description => 'Read error action.',
 753         optional => 1,
 754     },
 755 );
 756
 757 my %iothread_fmt = ( iothread => {
 758         type => 'boolean',
 759         format_description => 'off|on',
 760         description => "Whether to use iothreads for this drive",
 761         optional => 1,
 762 });
 763
 764 my %model_fmt = (
 765     model => {
 766         type => 'string',
 767         format => 'urlencoded',
 768         format_description => 'model',
 769         maxLength => 40*3, # *3 since it's %xx url enoded
 770         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 771         optional => 1,
 772     },
 773 );
 774
 775 my %queues_fmt = (
 776     queues => {
 777         type => 'integer',
 778         format_description => 'nbqueues',
 779         description => "Number of queues.",
 780         minimum => 2,
 781         optional => 1
 782     }
 783 );
 784
 785 my $add_throttle_desc = sub {
 786     my ($key, $type, $what, $size, $longsize) = @_;
 787     $drivedesc_base{$key} = {
 788         type => $type,
 789         format_description => $size,
 790         description => "Maximum $what speed in $longsize per second.",
 791         optional => 1,
 792     };
 793 };
 794 # throughput: (leaky bucket)
 795 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 796 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 797 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 798 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 799 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 800 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 801 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 802 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 803 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 804
 805 # pools: (pool of IO before throttling starts taking effect)
 806 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 807 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 808 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 809 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 810 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 811 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 812
 813 my $ide_fmt = {
 814     %drivedesc_base,
 815     %rerror_fmt,
 816     %model_fmt,
 817 };
 818
 819 my $idedesc = {
 820     optional => 1,
 821     type => 'string', format => $ide_fmt,
 822     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 823 };
 824 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 825
 826 my $scsi_fmt = {
 827     %drivedesc_base,
 828     %iothread_fmt,
 829     %queues_fmt,
 830 };
 831 my $scsidesc = {
 832     optional => 1,
 833     type => 'string', format => $scsi_fmt,
 834     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 835 };
 836 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 837
 838 my $sata_fmt = {
 839     %drivedesc_base,
 840     %rerror_fmt,
 841 };
 842 my $satadesc = {
 843     optional => 1,
 844     type => 'string', format => $sata_fmt,
 845     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 846 };
 847 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 848
 849 my $virtio_fmt = {
 850     %drivedesc_base,
 851     %iothread_fmt,
 852     %rerror_fmt,
 853 };
 854 my $virtiodesc = {
 855     optional => 1,
 856     type => 'string', format => $virtio_fmt,
 857     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 858 };
 859 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 860
 861 my $alldrive_fmt = {
 862     %drivedesc_base,
 863     %rerror_fmt,
 864     %iothread_fmt,
 865     %model_fmt,
 866     %queues_fmt,
 867 };
 868
 869 my $usb_fmt = {
 870     host => {
 871         default_key => 1,
 872         type => 'string', format => 'pve-qm-usb-device',
 873         format_description => 'HOSTUSBDEVICE|spice',
 874         description => 'The Host USB device or port or the value spice',
 875     },
 876     usb3 => {
 877         optional => 1,
 878         type => 'boolean',
 879         format_description => 'yes|no',
 880         description => 'Specifies whether if given host option is a USB3 device or port',
 881     },
 882 };
 883
 884 my $usbdesc = {
 885     optional => 1,
 886     type => 'string', format => $usb_fmt,
 887     description => <<EODESCR,
 888 Configure an USB device (n is 0 to 4). This can be used to
 889 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 890
 891 'bus-port(.port)*' (decimal numbers) or
 892 'vendor_id:product_id' (hexadeciaml numbers) or
 893 'spice'
 894
 895 You can use the 'lsusb -t' command to list existing usb devices.
 896
 897 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 898
 899 The value 'spice' can be used to add a usb redirection devices for spice.
 900
 901 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 902
 903 EODESCR
 904 };
 905 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 906
 907 # NOTE: the match-groups of this regex are used in parse_hostpci
 908 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 909 my $hostpci_fmt = {
 910     host => {
 911         default_key => 1,
 912         type => 'string',
 913         pattern => qr/$PCIRE(;$PCIRE)*/,
 914         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 915         description => "The PCI ID of a host's PCI device or a list of PCI virtual functions of the host.",
 916     },
 917     rombar => {
 918         type => 'boolean',
 919         optional => 1,
 920         default => 1,
 921     },
 922     pcie => {
 923         type => 'boolean',
 924         optional => 1,
 925         default => 0,
 926     },
 927     'x-vga' => {
 928         type => 'boolean',
 929         optional => 1,
 930         default => 0,
 931     },
 932 };
 933 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
 934
 935 my $hostpcidesc = {
 936         optional => 1,
 937         type => 'string', format => 'pve-qm-hostpci',
 938         description => <<EODESCR,
 939 Map host pci devices. HOSTPCIDEVICE syntax is:
 940
 941 'bus:dev.func' (hexadecimal numbers)
 942
 943 You can us the 'lspci' command to list existing pci devices.
 944
 945 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 946
 947 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 948
 949 Experimental: user reported problems with this option.
 950 EODESCR
 951 };
 952 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 953
 954 my $serialdesc = {
 955         optional => 1,
 956         type => 'string',
 957         pattern => '(/dev/.+|socket)',
 958         description =>  <<EODESCR,
 959 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 960
 961 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 962
 963 Experimental: user reported problems with this option.
 964 EODESCR
 965 };
 966
 967 my $paralleldesc= {
 968         optional => 1,
 969         type => 'string',
 970         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 971         description =>  <<EODESCR,
 972 Map host parallel devices (n is 0 to 2).
 973
 974 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 975
 976 Experimental: user reported problems with this option.
 977 EODESCR
 978 };
 979
 980 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 981     $confdesc->{"parallel$i"} = $paralleldesc;
 982 }
 983
 984 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 985     $confdesc->{"serial$i"} = $serialdesc;
 986 }
 987
 988 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 989     $confdesc->{"hostpci$i"} = $hostpcidesc;
 990 }
 991
 992 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 993     $drivename_hash->{"ide$i"} = 1;
 994     $confdesc->{"ide$i"} = $idedesc;
 995 }
 996
 997 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 998     $drivename_hash->{"sata$i"} = 1;
 999     $confdesc->{"sata$i"} = $satadesc;
1000 }
1001
1002 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1003     $drivename_hash->{"scsi$i"} = 1;
1004     $confdesc->{"scsi$i"} = $scsidesc ;
1005 }
1006
1007 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1008     $drivename_hash->{"virtio$i"} = 1;
1009     $confdesc->{"virtio$i"} = $virtiodesc;
1010 }
1011
1012 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1013     $confdesc->{"usb$i"} = $usbdesc;
1014 }
1015
1016 my $unuseddesc = {
1017     optional => 1,
1018     type => 'string', format => 'pve-volume-id',
1019     description => "Reference to unused volumes.",
1020 };
1021
1022 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1023     $confdesc->{"unused$i"} = $unuseddesc;
1024 }
1025
1026 my $kvm_api_version = 0;
1027
1028 sub kvm_version {
1029
1030     return $kvm_api_version if $kvm_api_version;
1031
1032     my $fh = IO::File->new("</dev/kvm") ||
1033         return 0;
1034
1035     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1036         $kvm_api_version = $v;
1037     }
1038
1039     $fh->close();
1040
1041     return  $kvm_api_version;
1042 }
1043
1044 my $kvm_user_version;
1045
1046 sub kvm_user_version {
1047
1048     return $kvm_user_version if $kvm_user_version;
1049
1050     $kvm_user_version = 'unknown';
1051
1052     my $code = sub {
1053         my $line = shift;
1054         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1055             $kvm_user_version = $2;
1056         }
1057     };
1058
1059     eval { run_command("kvm -version", outfunc => $code); };
1060     warn $@ if $@;
1061
1062     return $kvm_user_version;
1063
1064 }
1065
1066 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1067
1068 sub valid_drive_names {
1069     # order is important - used to autoselect boot disk
1070     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1071             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1072             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1073             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1074 }
1075
1076 sub is_valid_drivename {
1077     my $dev = shift;
1078
1079     return defined($drivename_hash->{$dev});
1080 }
1081
1082 sub option_exists {
1083     my $key = shift;
1084     return defined($confdesc->{$key});
1085 }
1086
1087 sub nic_models {
1088     return $nic_model_list;
1089 }
1090
1091 sub os_list_description {
1092
1093     return {
1094         other => 'Other',
1095         wxp => 'Windows XP',
1096         w2k => 'Windows 2000',
1097         w2k3 =>, 'Windows 2003',
1098         w2k8 => 'Windows 2008',
1099         wvista => 'Windows Vista',
1100         win7 => 'Windows 7',
1101         win8 => 'Windows 8/2012',
1102         l24 => 'Linux 2.4',
1103         l26 => 'Linux 2.6',
1104     };
1105 }
1106
1107 my $cdrom_path;
1108
1109 sub get_cdrom_path {
1110
1111     return  $cdrom_path if $cdrom_path;
1112
1113     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1114     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1115     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1116 }
1117
1118 sub get_iso_path {
1119     my ($storecfg, $vmid, $cdrom) = @_;
1120
1121     if ($cdrom eq 'cdrom') {
1122         return get_cdrom_path();
1123     } elsif ($cdrom eq 'none') {
1124         return '';
1125     } elsif ($cdrom =~ m|^/|) {
1126         return $cdrom;
1127     } else {
1128         return PVE::Storage::path($storecfg, $cdrom);
1129     }
1130 }
1131
1132 # try to convert old style file names to volume IDs
1133 sub filename_to_volume_id {
1134     my ($vmid, $file, $media) = @_;
1135
1136     if (!($file eq 'none' || $file eq 'cdrom' ||
1137           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1138
1139         return undef if $file =~ m|/|;
1140
1141         if ($media && $media eq 'cdrom') {
1142             $file = "local:iso/$file";
1143         } else {
1144             $file = "local:$vmid/$file";
1145         }
1146     }
1147
1148     return $file;
1149 }
1150
1151 sub verify_media_type {
1152     my ($opt, $vtype, $media) = @_;
1153
1154     return if !$media;
1155
1156     my $etype;
1157     if ($media eq 'disk') {
1158         $etype = 'images';
1159     } elsif ($media eq 'cdrom') {
1160         $etype = 'iso';
1161     } else {
1162         die "internal error";
1163     }
1164
1165     return if ($vtype eq $etype);
1166
1167     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1168 }
1169
1170 sub cleanup_drive_path {
1171     my ($opt, $storecfg, $drive) = @_;
1172
1173     # try to convert filesystem paths to volume IDs
1174
1175     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1176         ($drive->{file} !~ m|^/dev/.+|) &&
1177         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1178         ($drive->{file} !~ m/^\d+$/)) {
1179         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1180         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1181         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1182         verify_media_type($opt, $vtype, $drive->{media});
1183         $drive->{file} = $volid;
1184     }
1185
1186     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1187 }
1188
1189 sub parse_hotplug_features {
1190     my ($data) = @_;
1191
1192     my $res = {};
1193
1194     return $res if $data eq '0';
1195
1196     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1197
1198     foreach my $feature (PVE::Tools::split_list($data)) {
1199         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1200             $res->{$1} = 1;
1201         } else {
1202             warn "ignoring unknown hotplug feature '$feature'\n";
1203         }
1204     }
1205     return $res;
1206 }
1207
1208 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1209 sub pve_verify_hotplug_features {
1210     my ($value, $noerr) = @_;
1211
1212     return $value if parse_hotplug_features($value);
1213
1214     return undef if $noerr;
1215
1216     die "unable to parse hotplug option\n";
1217 }
1218
1219 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1220 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1221 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1222 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1223 #        [,iothread=on][,serial=serial][,model=model]
1224
1225 sub parse_drive {
1226     my ($key, $data) = @_;
1227
1228     my ($interface, $index);
1229
1230     if ($key =~ m/^([^\d]+)(\d+)$/) {
1231         $interface = $1;
1232         $index = $2;
1233     } else {
1234         return undef;
1235     }
1236
1237     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1238                                      : $confdesc->{$key}->{format};
1239     if (!$desc) {
1240         warn "invalid drive key: $key\n";
1241         return undef;
1242     }
1243     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1244     return undef if !$res;
1245     $res->{interface} = $interface;
1246     $res->{index} = $index;
1247
1248     my $error = 0;
1249     foreach my $opt (qw(bps bps_rd bps_wr)) {
1250         if (my $bps = defined(delete $res->{$opt})) {
1251             if (defined($res->{"m$opt"})) {
1252                 warn "both $opt and m$opt specified\n";
1253                 ++$error;
1254                 next;
1255             }
1256             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1257         }
1258     }
1259     return undef if $error;
1260
1261     return undef if $res->{mbps_rd} && $res->{mbps};
1262     return undef if $res->{mbps_wr} && $res->{mbps};
1263     return undef if $res->{iops_rd} && $res->{iops};
1264     return undef if $res->{iops_wr} && $res->{iops};
1265
1266     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1267         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1268         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1269         return undef if $res->{interface} eq 'virtio';
1270     }
1271
1272     if (my $size = $res->{size}) {
1273         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1274     }
1275
1276     return $res;
1277 }
1278
1279 sub print_drive {
1280     my ($vmid, $drive) = @_;
1281     my $data = { %$drive };
1282     delete $data->{$_} for qw(index interface);
1283     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1284 }
1285
1286 sub scsi_inquiry {
1287     my($fh, $noerr) = @_;
1288
1289     my $SG_IO = 0x2285;
1290     my $SG_GET_VERSION_NUM = 0x2282;
1291
1292     my $versionbuf = "\x00" x 8;
1293     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1294     if (!$ret) {
1295         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1296         return undef;
1297     }
1298     my $version = unpack("I", $versionbuf);
1299     if ($version < 30000) {
1300         die "scsi generic interface too old\n"  if !$noerr;
1301         return undef;
1302     }
1303
1304     my $buf = "\x00" x 36;
1305     my $sensebuf = "\x00" x 8;
1306     my $cmd = pack("C x3 C x1", 0x12, 36);
1307
1308     # see /usr/include/scsi/sg.h
1309     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1310
1311     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1312                       length($sensebuf), 0, length($buf), $buf,
1313                       $cmd, $sensebuf, 6000);
1314
1315     $ret = ioctl($fh, $SG_IO, $packet);
1316     if (!$ret) {
1317         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1318         return undef;
1319     }
1320
1321     my @res = unpack($sg_io_hdr_t, $packet);
1322     if ($res[17] || $res[18]) {
1323         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1324         return undef;
1325     }
1326
1327     my $res = {};
1328     (my $byte0, my $byte1, $res->{vendor},
1329      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1330
1331     $res->{removable} = $byte1 & 128 ? 1 : 0;
1332     $res->{type} = $byte0 & 31;
1333
1334     return $res;
1335 }
1336
1337 sub path_is_scsi {
1338     my ($path) = @_;
1339
1340     my $fh = IO::File->new("+<$path") || return undef;
1341     my $res = scsi_inquiry($fh, 1);
1342     close($fh);
1343
1344     return $res;
1345 }
1346
1347 sub machine_type_is_q35 {
1348     my ($conf) = @_;
1349
1350     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1351 }
1352
1353 sub print_tabletdevice_full {
1354     my ($conf) = @_;
1355
1356     my $q35 = machine_type_is_q35($conf);
1357
1358     # we use uhci for old VMs because tablet driver was buggy in older qemu
1359     my $usbbus = $q35 ? "ehci" : "uhci";
1360
1361     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1362 }
1363
1364 sub print_drivedevice_full {
1365     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1366
1367     my $device = '';
1368     my $maxdev = 0;
1369
1370     if ($drive->{interface} eq 'virtio') {
1371         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1372         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1373         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1374     } elsif ($drive->{interface} eq 'scsi') {
1375
1376         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1377         my $unit = $drive->{index} % $maxdev;
1378         my $devicetype = 'hd';
1379         my $path = '';
1380         if (drive_is_cdrom($drive)) {
1381             $devicetype = 'cd';
1382         } else {
1383             if ($drive->{file} =~ m|^/|) {
1384                 $path = $drive->{file};
1385                 if (my $info = path_is_scsi($path)) {
1386                     if ($info->{type} == 0) {
1387                         $devicetype = 'block';
1388                     } elsif ($info->{type} == 1) { # tape
1389                         $devicetype = 'generic';
1390                     }
1391                 }
1392             } else {
1393                  $path = PVE::Storage::path($storecfg, $drive->{file});
1394             }
1395
1396             if($path =~ m/^iscsi\:\/\//){
1397                 $devicetype = 'generic';
1398             }
1399         }
1400
1401         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1402             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1403         } else {
1404             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1405         }
1406
1407     } elsif ($drive->{interface} eq 'ide'){
1408         $maxdev = 2;
1409         my $controller = int($drive->{index} / $maxdev);
1410         my $unit = $drive->{index} % $maxdev;
1411         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1412
1413         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1414         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1415             $model = URI::Escape::uri_unescape($model);
1416             $device .= ",model=$model";
1417         }
1418     } elsif ($drive->{interface} eq 'sata'){
1419         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1420         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1421         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1422     } elsif ($drive->{interface} eq 'usb') {
1423         die "implement me";
1424         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1425     } else {
1426         die "unsupported interface type";
1427     }
1428
1429     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1430
1431     return $device;
1432 }
1433
1434 sub get_initiator_name {
1435     my $initiator;
1436
1437     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1438     while (defined(my $line = <$fh>)) {
1439         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1440         $initiator = $1;
1441         last;
1442     }
1443     $fh->close();
1444
1445     return $initiator;
1446 }
1447
1448 sub print_drive_full {
1449     my ($storecfg, $vmid, $drive) = @_;
1450
1451     my $path;
1452     my $volid = $drive->{file};
1453     my $format;
1454
1455     if (drive_is_cdrom($drive)) {
1456         $path = get_iso_path($storecfg, $vmid, $volid);
1457     } else {
1458         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1459         if ($storeid) {
1460             $path = PVE::Storage::path($storecfg, $volid);
1461             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1462             $format = qemu_img_format($scfg, $volname);
1463         } else {
1464             $path = $volid;
1465             $format = "raw";
1466         }
1467    }
1468
1469     my $opts = '';
1470     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1471     foreach my $o (@qemu_drive_options) {
1472         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1473     }
1474     if (my $serial = $drive->{serial}) {
1475         $serial = URI::Escape::uri_unescape($serial);
1476         $opts .= ",serial=$serial";
1477     }
1478
1479     $opts .= ",format=$format" if $format && !$drive->{format};
1480
1481     foreach my $o (qw(bps bps_rd bps_wr)) {
1482         my $v = $drive->{"m$o"};
1483         $opts .= ",$o=" . int($v*1024*1024) if $v;
1484     }
1485
1486     my $cache_direct = 0;
1487
1488     if (my $cache = $drive->{cache}) {
1489         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1490     } elsif (!drive_is_cdrom($drive)) {
1491         $opts .= ",cache=none";
1492         $cache_direct = 1;
1493     }
1494
1495     # aio native works only with O_DIRECT
1496     if (!$drive->{aio}) {
1497         if($cache_direct) {
1498             $opts .= ",aio=native";
1499         } else {
1500             $opts .= ",aio=threads";
1501         }
1502     }
1503
1504     if (!drive_is_cdrom($drive)) {
1505         my $detectzeroes;
1506         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1507             $detectzeroes = 'off';
1508         } elsif ($drive->{discard}) {
1509             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1510         } else {
1511             # This used to be our default with discard not being specified:
1512             $detectzeroes = 'on';
1513         }
1514         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1515     }
1516
1517     my $pathinfo = $path ? "file=$path," : '';
1518
1519     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1520 }
1521
1522 sub print_netdevice_full {
1523     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1524
1525     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1526
1527     my $device = $net->{model};
1528     if ($net->{model} eq 'virtio') {
1529          $device = 'virtio-net-pci';
1530      };
1531
1532     my $pciaddr = print_pci_addr("$netid", $bridges);
1533     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1534     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1535         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1536         my $vectors = $net->{queues} * 2 + 2;
1537         $tmpstr .= ",vectors=$vectors,mq=on";
1538     }
1539     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1540
1541     if ($use_old_bios_files) {
1542         my $romfile;
1543         if ($device eq 'virtio-net-pci') {
1544             $romfile = 'pxe-virtio.rom';
1545         } elsif ($device eq 'e1000') {
1546             $romfile = 'pxe-e1000.rom';
1547         } elsif ($device eq 'ne2k') {
1548             $romfile = 'pxe-ne2k_pci.rom';
1549         } elsif ($device eq 'pcnet') {
1550             $romfile = 'pxe-pcnet.rom';
1551         } elsif ($device eq 'rtl8139') {
1552             $romfile = 'pxe-rtl8139.rom';
1553         }
1554         $tmpstr .= ",romfile=$romfile" if $romfile;
1555     }
1556
1557     return $tmpstr;
1558 }
1559
1560 sub print_netdev_full {
1561     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1562
1563     my $i = '';
1564     if ($netid =~ m/^net(\d+)$/) {
1565         $i = int($1);
1566     }
1567
1568     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1569
1570     my $ifname = "tap${vmid}i$i";
1571
1572     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1573     die "interface name '$ifname' is too long (max 15 character)\n"
1574         if length($ifname) >= 16;
1575
1576     my $vhostparam = '';
1577     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1578
1579     my $vmname = $conf->{name} || "vm$vmid";
1580
1581     my $netdev = "";
1582     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1583
1584     if ($net->{bridge}) {
1585         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1586     } else {
1587         $netdev = "type=user,id=$netid,hostname=$vmname";
1588     }
1589
1590     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1591
1592     return $netdev;
1593 }
1594
1595 sub drive_is_cdrom {
1596     my ($drive) = @_;
1597
1598     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1599
1600 }
1601
1602 sub parse_number_sets {
1603     my ($set) = @_;
1604     my $res = [];
1605     foreach my $part (split(/;/, $set)) {
1606         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1607             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1608             push @$res, [ $1, $2 ];
1609         } else {
1610             die "invalid range: $part\n";
1611         }
1612     }
1613     return $res;
1614 }
1615
1616 sub parse_numa {
1617     my ($data) = @_;
1618
1619     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1620     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1621     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1622     return $res;
1623 }
1624
1625 sub parse_hostpci {
1626     my ($value) = @_;
1627
1628     return undef if !$value;
1629
1630     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1631
1632     my @idlist = split(/;/, $res->{host});
1633     delete $res->{host};
1634     foreach my $id (@idlist) {
1635         if ($id =~ /^$PCIRE$/) {
1636             push @{$res->{pciid}}, { id => $1, function => ($2//'0') };
1637         } else {
1638             # should have been caught by parse_property_string already
1639             die "failed to parse PCI id: $id\n";
1640         }
1641     }
1642     return $res;
1643 }
1644
1645 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1646 sub parse_net {
1647     my ($data) = @_;
1648
1649     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1650     if ($@) {
1651         warn $@;
1652         return undef;
1653     }
1654     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1655     return $res;
1656 }
1657
1658 sub print_net {
1659     my $net = shift;
1660
1661     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1662 }
1663
1664 sub add_random_macs {
1665     my ($settings) = @_;
1666
1667     foreach my $opt (keys %$settings) {
1668         next if $opt !~ m/^net(\d+)$/;
1669         my $net = parse_net($settings->{$opt});
1670         next if !$net;
1671         $settings->{$opt} = print_net($net);
1672     }
1673 }
1674
1675 sub vm_is_volid_owner {
1676     my ($storecfg, $vmid, $volid) = @_;
1677
1678     if ($volid !~  m|^/|) {
1679         my ($path, $owner);
1680         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1681         if ($owner && ($owner == $vmid)) {
1682             return 1;
1683         }
1684     }
1685
1686     return undef;
1687 }
1688
1689 sub split_flagged_list {
1690     my $text = shift || '';
1691     $text =~ s/[,;]/ /g;
1692     $text =~ s/^\s+//;
1693     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1694 }
1695
1696 sub join_flagged_list {
1697     my ($how, $lst) = @_;
1698     join $how, map { $lst->{$_} . $_ } keys %$lst;
1699 }
1700
1701 sub vmconfig_delete_pending_option {
1702     my ($conf, $key, $force) = @_;
1703
1704     delete $conf->{pending}->{$key};
1705     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1706     $pending_delete_hash->{$key} = $force ? '!' : '';
1707     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1708 }
1709
1710 sub vmconfig_undelete_pending_option {
1711     my ($conf, $key) = @_;
1712
1713     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1714     delete $pending_delete_hash->{$key};
1715
1716     if (%$pending_delete_hash) {
1717         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1718     } else {
1719         delete $conf->{pending}->{delete};
1720     }
1721 }
1722
1723 sub vmconfig_register_unused_drive {
1724     my ($storecfg, $vmid, $conf, $drive) = @_;
1725
1726     if (!drive_is_cdrom($drive)) {
1727         my $volid = $drive->{file};
1728         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1729             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1730         }
1731     }
1732 }
1733
1734 sub vmconfig_cleanup_pending {
1735     my ($conf) = @_;
1736
1737     # remove pending changes when nothing changed
1738     my $changes;
1739     foreach my $opt (keys %{$conf->{pending}}) {
1740         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1741             $changes = 1;
1742             delete $conf->{pending}->{$opt};
1743         }
1744     }
1745
1746     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1747     my $pending_delete_hash = {};
1748     while (my ($opt, $force) = each %$current_delete_hash) {
1749         if (defined($conf->{$opt})) {
1750             $pending_delete_hash->{$opt} = $force;
1751         } else {
1752             $changes = 1;
1753         }
1754     }
1755
1756     if (%$pending_delete_hash) {
1757         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1758     } else {
1759         delete $conf->{pending}->{delete};
1760     }
1761
1762     return $changes;
1763 }
1764
1765 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1766 my $smbios1_fmt = {
1767     uuid => {
1768         type => 'string',
1769         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1770         format_description => 'UUID',
1771         optional => 1,
1772     },
1773     version => {
1774         type => 'string',
1775         pattern => '\S+',
1776         format_description => 'str',
1777         optional => 1,
1778     },
1779     serial => {
1780         type => 'string',
1781         pattern => '\S+',
1782         format_description => 'str',
1783         optional => 1,
1784     },
1785     manufacturer => {
1786         type => 'string',
1787         pattern => '\S+',
1788         format_description => 'name',
1789         optional => 1,
1790     },
1791     product => {
1792         type => 'string',
1793         pattern => '\S+',
1794         format_description => 'name',
1795         optional => 1,
1796     },
1797     sku => {
1798         type => 'string',
1799         pattern => '\S+',
1800         format_description => 'str',
1801         optional => 1,
1802     },
1803     family => {
1804         type => 'string',
1805         pattern => '\S+',
1806         format_description => 'str',
1807         optional => 1,
1808     },
1809 };
1810
1811 sub parse_smbios1 {
1812     my ($data) = @_;
1813
1814     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1815     warn $@ if $@;
1816     return $res;
1817 }
1818
1819 sub print_smbios1 {
1820     my ($smbios1) = @_;
1821     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1822 }
1823
1824 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1825
1826 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1827 sub verify_bootdisk {
1828     my ($value, $noerr) = @_;
1829
1830     return $value if is_valid_drivename($value);
1831
1832     return undef if $noerr;
1833
1834     die "invalid boot disk '$value'\n";
1835 }
1836
1837 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1838 sub verify_net {
1839     my ($value, $noerr) = @_;
1840
1841     return $value if parse_net($value);
1842
1843     return undef if $noerr;
1844
1845     die "unable to parse network options\n";
1846 }
1847
1848 sub parse_watchdog {
1849     my ($value) = @_;
1850
1851     return undef if !$value;
1852
1853     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1854     warn $@ if $@;
1855     return $res;
1856 }
1857
1858 sub parse_usb_device {
1859     my ($value) = @_;
1860
1861     return undef if !$value;
1862
1863     my $res = {};
1864     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1865         $res->{vendorid} = $2;
1866         $res->{productid} = $4;
1867     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1868         $res->{hostbus} = $1;
1869         $res->{hostport} = $2;
1870     } elsif ($value =~ m/^spice$/i) {
1871         $res->{spice} = 1;
1872     } else {
1873         return undef;
1874     }
1875
1876     return $res;
1877 }
1878
1879 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1880 sub verify_usb_device {
1881     my ($value, $noerr) = @_;
1882
1883     return $value if parse_usb_device($value);
1884
1885     return undef if $noerr;
1886
1887     die "unable to parse usb device\n";
1888 }
1889
1890 # add JSON properties for create and set function
1891 sub json_config_properties {
1892     my $prop = shift;
1893
1894     foreach my $opt (keys %$confdesc) {
1895         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1896         $prop->{$opt} = $confdesc->{$opt};
1897     }
1898
1899     return $prop;
1900 }
1901
1902 sub check_type {
1903     my ($key, $value) = @_;
1904
1905     die "unknown setting '$key'\n" if !$confdesc->{$key};
1906
1907     my $type = $confdesc->{$key}->{type};
1908
1909     if (!defined($value)) {
1910         die "got undefined value\n";
1911     }
1912
1913     if ($value =~ m/[\n\r]/) {
1914         die "property contains a line feed\n";
1915     }
1916
1917     if ($type eq 'boolean') {
1918         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1919         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1920         die "type check ('boolean') failed - got '$value'\n";
1921     } elsif ($type eq 'integer') {
1922         return int($1) if $value =~ m/^(\d+)$/;
1923         die "type check ('integer') failed - got '$value'\n";
1924     } elsif ($type eq 'number') {
1925         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1926         die "type check ('number') failed - got '$value'\n";
1927     } elsif ($type eq 'string') {
1928         if (my $fmt = $confdesc->{$key}->{format}) {
1929             if ($fmt eq 'pve-qm-drive') {
1930                 # special case - we need to pass $key to parse_drive()
1931                 my $drive = parse_drive($key, $value);
1932                 return $value if $drive;
1933                 die "unable to parse drive options\n";
1934             }
1935             PVE::JSONSchema::check_format($fmt, $value);
1936             return $value;
1937         }
1938         $value =~ s/^\"(.*)\"$/$1/;
1939         return $value;
1940     } else {
1941         die "internal error"
1942     }
1943 }
1944
1945 sub check_iommu_support{
1946     #fixme : need to check IOMMU support
1947     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1948
1949     my $iommu=1;
1950     return $iommu;
1951
1952 }
1953
1954 sub touch_config {
1955     my ($vmid) = @_;
1956
1957     my $conf = PVE::QemuConfig->config_file($vmid);
1958     utime undef, undef, $conf;
1959 }
1960
1961 sub destroy_vm {
1962     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1963
1964     my $conffile = PVE::QemuConfig->config_file($vmid);
1965
1966     my $conf = PVE::QemuConfig->load_config($vmid);
1967
1968     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1969
1970     # only remove disks owned by this VM
1971     foreach_drive($conf, sub {
1972         my ($ds, $drive) = @_;
1973
1974         return if drive_is_cdrom($drive);
1975
1976         my $volid = $drive->{file};
1977
1978         return if !$volid || $volid =~ m|^/|;
1979
1980         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1981         return if !$path || !$owner || ($owner != $vmid);
1982
1983         PVE::Storage::vdisk_free($storecfg, $volid);
1984     });
1985
1986     if ($keep_empty_config) {
1987         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1988     } else {
1989         unlink $conffile;
1990     }
1991
1992     # also remove unused disk
1993     eval {
1994         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1995
1996         eval {
1997             PVE::Storage::foreach_volid($dl, sub {
1998                 my ($volid, $sid, $volname, $d) = @_;
1999                 PVE::Storage::vdisk_free($storecfg, $volid);
2000             });
2001         };
2002         warn $@ if $@;
2003
2004     };
2005     warn $@ if $@;
2006 }
2007
2008 sub parse_vm_config {
2009     my ($filename, $raw) = @_;
2010
2011     return undef if !defined($raw);
2012
2013     my $res = {
2014         digest => Digest::SHA::sha1_hex($raw),
2015         snapshots => {},
2016         pending => {},
2017     };
2018
2019     $filename =~ m|/qemu-server/(\d+)\.conf$|
2020         || die "got strange filename '$filename'";
2021
2022     my $vmid = $1;
2023
2024     my $conf = $res;
2025     my $descr;
2026     my $section = '';
2027
2028     my @lines = split(/\n/, $raw);
2029     foreach my $line (@lines) {
2030         next if $line =~ m/^\s*$/;
2031
2032         if ($line =~ m/^\[PENDING\]\s*$/i) {
2033             $section = 'pending';
2034             if (defined($descr)) {
2035                 $descr =~ s/\s+$//;
2036                 $conf->{description} = $descr;
2037             }
2038             $descr = undef;
2039             $conf = $res->{$section} = {};
2040             next;
2041
2042         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2043             $section = $1;
2044             if (defined($descr)) {
2045                 $descr =~ s/\s+$//;
2046                 $conf->{description} = $descr;
2047             }
2048             $descr = undef;
2049             $conf = $res->{snapshots}->{$section} = {};
2050             next;
2051         }
2052
2053         if ($line =~ m/^\#(.*)\s*$/) {
2054             $descr = '' if !defined($descr);
2055             $descr .= PVE::Tools::decode_text($1) . "\n";
2056             next;
2057         }
2058
2059         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2060             $descr = '' if !defined($descr);
2061             $descr .= PVE::Tools::decode_text($2);
2062         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2063             $conf->{snapstate} = $1;
2064         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2065             my $key = $1;
2066             my $value = $2;
2067             $conf->{$key} = $value;
2068         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2069             my $value = $1;
2070             if ($section eq 'pending') {
2071                 $conf->{delete} = $value; # we parse this later
2072             } else {
2073                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2074             }
2075         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2076             my $key = $1;
2077             my $value = $2;
2078             eval { $value = check_type($key, $value); };
2079             if ($@) {
2080                 warn "vm $vmid - unable to parse value of '$key' - $@";
2081             } else {
2082                 my $fmt = $confdesc->{$key}->{format};
2083                 if ($fmt && $fmt eq 'pve-qm-drive') {
2084                     my $v = parse_drive($key, $value);
2085                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2086                         $v->{file} = $volid;
2087                         $value = print_drive($vmid, $v);
2088                     } else {
2089                         warn "vm $vmid - unable to parse value of '$key'\n";
2090                         next;
2091                     }
2092                 }
2093
2094                 if ($key eq 'cdrom') {
2095                     $conf->{ide2} = $value;
2096                 } else {
2097                     $conf->{$key} = $value;
2098                 }
2099             }
2100         }
2101     }
2102
2103     if (defined($descr)) {
2104         $descr =~ s/\s+$//;
2105         $conf->{description} = $descr;
2106     }
2107     delete $res->{snapstate}; # just to be sure
2108
2109     return $res;
2110 }
2111
2112 sub write_vm_config {
2113     my ($filename, $conf) = @_;
2114
2115     delete $conf->{snapstate}; # just to be sure
2116
2117     if ($conf->{cdrom}) {
2118         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2119         $conf->{ide2} = $conf->{cdrom};
2120         delete $conf->{cdrom};
2121     }
2122
2123     # we do not use 'smp' any longer
2124     if ($conf->{sockets}) {
2125         delete $conf->{smp};
2126     } elsif ($conf->{smp}) {
2127         $conf->{sockets} = $conf->{smp};
2128         delete $conf->{cores};
2129         delete $conf->{smp};
2130     }
2131
2132     my $used_volids = {};
2133
2134     my $cleanup_config = sub {
2135         my ($cref, $pending, $snapname) = @_;
2136
2137         foreach my $key (keys %$cref) {
2138             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2139                 $key eq 'snapstate' || $key eq 'pending';
2140             my $value = $cref->{$key};
2141             if ($key eq 'delete') {
2142                 die "propertry 'delete' is only allowed in [PENDING]\n"
2143                     if !$pending;
2144                 # fixme: check syntax?
2145                 next;
2146             }
2147             eval { $value = check_type($key, $value); };
2148             die "unable to parse value of '$key' - $@" if $@;
2149
2150             $cref->{$key} = $value;
2151
2152             if (!$snapname && is_valid_drivename($key)) {
2153                 my $drive = parse_drive($key, $value);
2154                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2155             }
2156         }
2157     };
2158
2159     &$cleanup_config($conf);
2160
2161     &$cleanup_config($conf->{pending}, 1);
2162
2163     foreach my $snapname (keys %{$conf->{snapshots}}) {
2164         die "internal error" if $snapname eq 'pending';
2165         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2166     }
2167
2168     # remove 'unusedX' settings if we re-add a volume
2169     foreach my $key (keys %$conf) {
2170         my $value = $conf->{$key};
2171         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2172             delete $conf->{$key};
2173         }
2174     }
2175
2176     my $generate_raw_config = sub {
2177         my ($conf, $pending) = @_;
2178
2179         my $raw = '';
2180
2181         # add description as comment to top of file
2182         if (defined(my $descr = $conf->{description})) {
2183             if ($descr) {
2184                 foreach my $cl (split(/\n/, $descr)) {
2185                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2186                 }
2187             } else {
2188                 $raw .= "#\n" if $pending;
2189             }
2190         }
2191
2192         foreach my $key (sort keys %$conf) {
2193             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2194             $raw .= "$key: $conf->{$key}\n";
2195         }
2196         return $raw;
2197     };
2198
2199     my $raw = &$generate_raw_config($conf);
2200
2201     if (scalar(keys %{$conf->{pending}})){
2202         $raw .= "\n[PENDING]\n";
2203         $raw .= &$generate_raw_config($conf->{pending}, 1);
2204     }
2205
2206     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2207         $raw .= "\n[$snapname]\n";
2208         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2209     }
2210
2211     return $raw;
2212 }
2213
2214 sub load_defaults {
2215
2216     my $res = {};
2217
2218     # we use static defaults from our JSON schema configuration
2219     foreach my $key (keys %$confdesc) {
2220         if (defined(my $default = $confdesc->{$key}->{default})) {
2221             $res->{$key} = $default;
2222         }
2223     }
2224
2225     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2226     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2227
2228     return $res;
2229 }
2230
2231 sub config_list {
2232     my $vmlist = PVE::Cluster::get_vmlist();
2233     my $res = {};
2234     return $res if !$vmlist || !$vmlist->{ids};
2235     my $ids = $vmlist->{ids};
2236
2237     foreach my $vmid (keys %$ids) {
2238         my $d = $ids->{$vmid};
2239         next if !$d->{node} || $d->{node} ne $nodename;
2240         next if !$d->{type} || $d->{type} ne 'qemu';
2241         $res->{$vmid}->{exists} = 1;
2242     }
2243     return $res;
2244 }
2245
2246 # test if VM uses local resources (to prevent migration)
2247 sub check_local_resources {
2248     my ($conf, $noerr) = @_;
2249
2250     my $loc_res = 0;
2251
2252     $loc_res = 1 if $conf->{hostusb}; # old syntax
2253     $loc_res = 1 if $conf->{hostpci}; # old syntax
2254
2255     foreach my $k (keys %$conf) {
2256         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2257         # sockets are safe: they will recreated be on the target side post-migrate
2258         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2259         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2260     }
2261
2262     die "VM uses local resources\n" if $loc_res && !$noerr;
2263
2264     return $loc_res;
2265 }
2266
2267 # check if used storages are available on all nodes (use by migrate)
2268 sub check_storage_availability {
2269     my ($storecfg, $conf, $node) = @_;
2270
2271     foreach_drive($conf, sub {
2272         my ($ds, $drive) = @_;
2273
2274         my $volid = $drive->{file};
2275         return if !$volid;
2276
2277         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2278         return if !$sid;
2279
2280         # check if storage is available on both nodes
2281         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2282         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2283    });
2284 }
2285
2286 # list nodes where all VM images are available (used by has_feature API)
2287 sub shared_nodes {
2288     my ($conf, $storecfg) = @_;
2289
2290     my $nodelist = PVE::Cluster::get_nodelist();
2291     my $nodehash = { map { $_ => 1 } @$nodelist };
2292     my $nodename = PVE::INotify::nodename();
2293
2294     foreach_drive($conf, sub {
2295         my ($ds, $drive) = @_;
2296
2297         my $volid = $drive->{file};
2298         return if !$volid;
2299
2300         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2301         if ($storeid) {
2302             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2303             if ($scfg->{disable}) {
2304                 $nodehash = {};
2305             } elsif (my $avail = $scfg->{nodes}) {
2306                 foreach my $node (keys %$nodehash) {
2307                     delete $nodehash->{$node} if !$avail->{$node};
2308                 }
2309             } elsif (!$scfg->{shared}) {
2310                 foreach my $node (keys %$nodehash) {
2311                     delete $nodehash->{$node} if $node ne $nodename
2312                 }
2313             }
2314         }
2315     });
2316
2317     return $nodehash
2318 }
2319
2320 sub check_cmdline {
2321     my ($pidfile, $pid) = @_;
2322
2323     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2324     if (defined($fh)) {
2325         my $line = <$fh>;
2326         $fh->close;
2327         return undef if !$line;
2328         my @param = split(/\0/, $line);
2329
2330         my $cmd = $param[0];
2331         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2332
2333         for (my $i = 0; $i < scalar (@param); $i++) {
2334             my $p = $param[$i];
2335             next if !$p;
2336             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2337                 my $p = $param[$i+1];
2338                 return 1 if $p && ($p eq $pidfile);
2339                 return undef;
2340             }
2341         }
2342     }
2343     return undef;
2344 }
2345
2346 sub check_running {
2347     my ($vmid, $nocheck, $node) = @_;
2348
2349     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2350
2351     die "unable to find configuration file for VM $vmid - no such machine\n"
2352         if !$nocheck && ! -f $filename;
2353
2354     my $pidfile = pidfile_name($vmid);
2355
2356     if (my $fd = IO::File->new("<$pidfile")) {
2357         my $st = stat($fd);
2358         my $line = <$fd>;
2359         close($fd);
2360
2361         my $mtime = $st->mtime;
2362         if ($mtime > time()) {
2363             warn "file '$filename' modified in future\n";
2364         }
2365
2366         if ($line =~ m/^(\d+)$/) {
2367             my $pid = $1;
2368             if (check_cmdline($pidfile, $pid)) {
2369                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2370                     return $pid;
2371                 }
2372             }
2373         }
2374     }
2375
2376     return undef;
2377 }
2378
2379 sub vzlist {
2380
2381     my $vzlist = config_list();
2382
2383     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2384
2385     while (defined(my $de = $fd->read)) {
2386         next if $de !~ m/^(\d+)\.pid$/;
2387         my $vmid = $1;
2388         next if !defined($vzlist->{$vmid});
2389         if (my $pid = check_running($vmid)) {
2390             $vzlist->{$vmid}->{pid} = $pid;
2391         }
2392     }
2393
2394     return $vzlist;
2395 }
2396
2397 sub disksize {
2398     my ($storecfg, $conf) = @_;
2399
2400     my $bootdisk = $conf->{bootdisk};
2401     return undef if !$bootdisk;
2402     return undef if !is_valid_drivename($bootdisk);
2403
2404     return undef if !$conf->{$bootdisk};
2405
2406     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2407     return undef if !defined($drive);
2408
2409     return undef if drive_is_cdrom($drive);
2410
2411     my $volid = $drive->{file};
2412     return undef if !$volid;
2413
2414     return $drive->{size};
2415 }
2416
2417 my $last_proc_pid_stat;
2418
2419 # get VM status information
2420 # This must be fast and should not block ($full == false)
2421 # We only query KVM using QMP if $full == true (this can be slow)
2422 sub vmstatus {
2423     my ($opt_vmid, $full) = @_;
2424
2425     my $res = {};
2426
2427     my $storecfg = PVE::Storage::config();
2428
2429     my $list = vzlist();
2430     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2431
2432     my $cpucount = $cpuinfo->{cpus} || 1;
2433
2434     foreach my $vmid (keys %$list) {
2435         next if $opt_vmid && ($vmid ne $opt_vmid);
2436
2437         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2438         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2439
2440         my $d = {};
2441         $d->{pid} = $list->{$vmid}->{pid};
2442
2443         # fixme: better status?
2444         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2445
2446         my $size = disksize($storecfg, $conf);
2447         if (defined($size)) {
2448             $d->{disk} = 0; # no info available
2449             $d->{maxdisk} = $size;
2450         } else {
2451             $d->{disk} = 0;
2452             $d->{maxdisk} = 0;
2453         }
2454
2455         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2456         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2457         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2458
2459         $d->{name} = $conf->{name} || "VM $vmid";
2460         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2461
2462         if ($conf->{balloon}) {
2463             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2464             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2465         }
2466
2467         $d->{uptime} = 0;
2468         $d->{cpu} = 0;
2469         $d->{mem} = 0;
2470
2471         $d->{netout} = 0;
2472         $d->{netin} = 0;
2473
2474         $d->{diskread} = 0;
2475         $d->{diskwrite} = 0;
2476
2477         $d->{template} = PVE::QemuConfig->is_template($conf);
2478
2479         $res->{$vmid} = $d;
2480     }
2481
2482     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2483     foreach my $dev (keys %$netdev) {
2484         next if $dev !~ m/^tap([1-9]\d*)i/;
2485         my $vmid = $1;
2486         my $d = $res->{$vmid};
2487         next if !$d;
2488
2489         $d->{netout} += $netdev->{$dev}->{receive};
2490         $d->{netin} += $netdev->{$dev}->{transmit};
2491
2492         if ($full) {
2493             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2494             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2495         }
2496
2497     }
2498
2499     my $ctime = gettimeofday;
2500
2501     foreach my $vmid (keys %$list) {
2502
2503         my $d = $res->{$vmid};
2504         my $pid = $d->{pid};
2505         next if !$pid;
2506
2507         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2508         next if !$pstat; # not running
2509
2510         my $used = $pstat->{utime} + $pstat->{stime};
2511
2512         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2513
2514         if ($pstat->{vsize}) {
2515             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2516         }
2517
2518         my $old = $last_proc_pid_stat->{$pid};
2519         if (!$old) {
2520             $last_proc_pid_stat->{$pid} = {
2521                 time => $ctime,
2522                 used => $used,
2523                 cpu => 0,
2524             };
2525             next;
2526         }
2527
2528         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2529
2530         if ($dtime > 1000) {
2531             my $dutime = $used -  $old->{used};
2532
2533             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2534             $last_proc_pid_stat->{$pid} = {
2535                 time => $ctime,
2536                 used => $used,
2537                 cpu => $d->{cpu},
2538             };
2539         } else {
2540             $d->{cpu} = $old->{cpu};
2541         }
2542     }
2543
2544     return $res if !$full;
2545
2546     my $qmpclient = PVE::QMPClient->new();
2547
2548     my $ballooncb = sub {
2549         my ($vmid, $resp) = @_;
2550
2551         my $info = $resp->{'return'};
2552         return if !$info->{max_mem};
2553
2554         my $d = $res->{$vmid};
2555
2556         # use memory assigned to VM
2557         $d->{maxmem} = $info->{max_mem};
2558         $d->{balloon} = $info->{actual};
2559
2560         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2561             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2562             $d->{freemem} = $info->{free_mem};
2563         }
2564
2565         $d->{ballooninfo} = $info;
2566     };
2567
2568     my $blockstatscb = sub {
2569         my ($vmid, $resp) = @_;
2570         my $data = $resp->{'return'} || [];
2571         my $totalrdbytes = 0;
2572         my $totalwrbytes = 0;
2573
2574         for my $blockstat (@$data) {
2575             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2576             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2577
2578             $blockstat->{device} =~ s/drive-//;
2579             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2580         }
2581         $res->{$vmid}->{diskread} = $totalrdbytes;
2582         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2583     };
2584
2585     my $statuscb = sub {
2586         my ($vmid, $resp) = @_;
2587
2588         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2589         # this fails if ballon driver is not loaded, so this must be
2590         # the last commnand (following command are aborted if this fails).
2591         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2592
2593         my $status = 'unknown';
2594         if (!defined($status = $resp->{'return'}->{status})) {
2595             warn "unable to get VM status\n";
2596             return;
2597         }
2598
2599         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2600     };
2601
2602     foreach my $vmid (keys %$list) {
2603         next if $opt_vmid && ($vmid ne $opt_vmid);
2604         next if !$res->{$vmid}->{pid}; # not running
2605         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2606     }
2607
2608     $qmpclient->queue_execute(undef, 1);
2609
2610     foreach my $vmid (keys %$list) {
2611         next if $opt_vmid && ($vmid ne $opt_vmid);
2612         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2613     }
2614
2615     return $res;
2616 }
2617
2618 sub foreach_dimm {
2619     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2620
2621     my $dimm_id = 0;
2622     my $current_size = 1024;
2623     my $dimm_size = 512;
2624     return if $current_size == $memory;
2625
2626     for (my $j = 0; $j < 8; $j++) {
2627         for (my $i = 0; $i < 32; $i++) {
2628             my $name = "dimm${dimm_id}";
2629             $dimm_id++;
2630             my $numanode = $i % $sockets;
2631             $current_size += $dimm_size;
2632             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2633             return  $current_size if $current_size >= $memory;
2634         }
2635         $dimm_size *= 2;
2636     }
2637 }
2638
2639 sub foreach_reverse_dimm {
2640     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2641
2642     my $dimm_id = 253;
2643     my $current_size = 4177920;
2644     my $dimm_size = 65536;
2645     return if $current_size == $memory;
2646
2647     for (my $j = 0; $j < 8; $j++) {
2648         for (my $i = 0; $i < 32; $i++) {
2649             my $name = "dimm${dimm_id}";
2650             $dimm_id--;
2651             my $numanode = $i % $sockets;
2652             $current_size -= $dimm_size;
2653             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2654             return  $current_size if $current_size <= $memory;
2655         }
2656         $dimm_size /= 2;
2657     }
2658 }
2659
2660 sub foreach_drive {
2661     my ($conf, $func) = @_;
2662
2663     foreach my $ds (valid_drive_names()) {
2664         next if !defined($conf->{$ds});
2665
2666         my $drive = parse_drive($ds, $conf->{$ds});
2667         next if !$drive;
2668
2669         &$func($ds, $drive);
2670     }
2671 }
2672
2673 sub foreach_volid {
2674     my ($conf, $func) = @_;
2675
2676     my $volhash = {};
2677
2678     my $test_volid = sub {
2679         my ($volid, $is_cdrom) = @_;
2680
2681         return if !$volid;
2682
2683         $volhash->{$volid} = $is_cdrom || 0;
2684     };
2685
2686     foreach_drive($conf, sub {
2687         my ($ds, $drive) = @_;
2688         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2689     });
2690
2691     foreach my $snapname (keys %{$conf->{snapshots}}) {
2692         my $snap = $conf->{snapshots}->{$snapname};
2693         &$test_volid($snap->{vmstate}, 0);
2694         foreach_drive($snap, sub {
2695             my ($ds, $drive) = @_;
2696             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2697         });
2698     }
2699
2700     foreach my $volid (keys %$volhash) {
2701         &$func($volid, $volhash->{$volid});
2702     }
2703 }
2704
2705 sub vga_conf_has_spice {
2706     my ($vga) = @_;
2707
2708     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2709
2710     return $1 || 1;
2711 }
2712
2713 sub config_to_command {
2714     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2715
2716     my $cmd = [];
2717     my $globalFlags = [];
2718     my $machineFlags = [];
2719     my $rtcFlags = [];
2720     my $cpuFlags = [];
2721     my $devices = [];
2722     my $pciaddr = '';
2723     my $bridges = {};
2724     my $kvmver = kvm_user_version();
2725     my $vernum = 0; # unknown
2726     my $ostype = $conf->{ostype};
2727     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2728         $vernum = $1*1000000+$2*1000;
2729     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2730         $vernum = $1*1000000+$2*1000+$3;
2731     }
2732
2733     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2734
2735     my $have_ovz = -f '/proc/vz/vestat';
2736
2737     my $q35 = machine_type_is_q35($conf);
2738     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2739     my $machine_type = $forcemachine || $conf->{machine};
2740     my $use_old_bios_files = undef;
2741     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2742
2743     my $cpuunits = defined($conf->{cpuunits}) ?
2744             $conf->{cpuunits} : $defaults->{cpuunits};
2745
2746     push @$cmd, '/usr/bin/systemd-run';
2747     push @$cmd, '--scope';
2748     push @$cmd, '--slice', "qemu";
2749     push @$cmd, '--unit', $vmid;
2750     # set KillMode=none, so that systemd don't kill those scopes
2751     # at shutdown (pve-manager service should stop the VMs instead)
2752     push @$cmd, '-p', "KillMode=none";
2753     push @$cmd, '-p', "CPUShares=$cpuunits";
2754     if ($conf->{cpulimit}) {
2755         my $cpulimit = int($conf->{cpulimit} * 100);
2756         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2757     }
2758
2759     push @$cmd, '/usr/bin/kvm';
2760
2761     push @$cmd, '-id', $vmid;
2762
2763     my $use_virtio = 0;
2764
2765     my $qmpsocket = qmp_socket($vmid);
2766     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2767     push @$cmd, '-mon', "chardev=qmp,mode=control";
2768
2769
2770     push @$cmd, '-pidfile' , pidfile_name($vmid);
2771
2772     push @$cmd, '-daemonize';
2773
2774     if ($conf->{smbios1}) {
2775         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2776     }
2777
2778     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2779         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2780         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2781         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2782         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2783         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2784         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2785     }
2786
2787     if ($q35) {
2788         # the q35 chipset support native usb2, so we enable usb controller
2789         # by default for this machine type
2790         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2791     } else {
2792         $pciaddr = print_pci_addr("piix3", $bridges);
2793         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2794
2795         my $use_usb2 = 0;
2796         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2797             next if !$conf->{"usb$i"};
2798             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2799             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2800             $use_usb2 = 1;
2801         }
2802         # include usb device config
2803         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2804     }
2805
2806     # add usb3 controller if needed
2807
2808     my $use_usb3 = 0;
2809     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2810         next if !$conf->{"usb$i"};
2811         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2812         next if !$d || !$d->{usb3};
2813         $use_usb3 = 1;
2814     }
2815
2816     $pciaddr = print_pci_addr("xhci", $bridges);
2817     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2818
2819     my $vga = $conf->{vga};
2820
2821     my $qxlnum = vga_conf_has_spice($vga);
2822     $vga = 'qxl' if $qxlnum;
2823
2824     if (!$vga) {
2825         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2826                                 $conf->{ostype} eq 'win7' ||
2827                                 $conf->{ostype} eq 'w2k8')) {
2828             $vga = 'std';
2829         } else {
2830             $vga = 'cirrus';
2831         }
2832     }
2833
2834     # enable absolute mouse coordinates (needed by vnc)
2835     my $tablet;
2836     if (defined($conf->{tablet})) {
2837         $tablet = $conf->{tablet};
2838     } else {
2839         $tablet = $defaults->{tablet};
2840         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2841         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2842     }
2843
2844     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2845
2846     my $kvm_off = 0;
2847     # host pci devices
2848     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2849         my $d = parse_hostpci($conf->{"hostpci$i"});
2850         next if !$d;
2851
2852         my $pcie = $d->{pcie};
2853         if($pcie){
2854             die "q35 machine model is not enabled" if !$q35;
2855             $pciaddr = print_pcie_addr("hostpci$i");
2856         }else{
2857             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2858         }
2859
2860         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2861         my $xvga = '';
2862         if ($d->{'x-vga'}) {
2863             $xvga = ',x-vga=on';
2864             $kvm_off = 1;
2865             $vga = 'none';
2866             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2867                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2868             }
2869             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2870                 $xvga = "";
2871             }
2872         }
2873         my $pcidevices = $d->{pciid};
2874         my $multifunction = 1 if @$pcidevices > 1;
2875
2876         my $j=0;
2877         foreach my $pcidevice (@$pcidevices) {
2878
2879             my $id = "hostpci$i";
2880             $id .= ".$j" if $multifunction;
2881             my $addr = $pciaddr;
2882             $addr .= ".$j" if $multifunction;
2883             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2884
2885             if($j == 0){
2886                 $devicestr .= "$rombar$xvga";
2887                 $devicestr .= ",multifunction=on" if $multifunction;
2888             }
2889
2890             push @$devices, '-device', $devicestr;
2891             $j++;
2892         }
2893     }
2894
2895     # usb devices
2896     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2897         next if !$conf->{"usb$i"};
2898         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2899         next if !$d;
2900
2901         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2902         my $usbbus = '';
2903         if (defined($d->{usb3}) && $d->{usb3}) {
2904             $usbbus = ',bus=xhci.0';
2905         }
2906
2907         if (defined($d->{host})) {
2908             $d = parse_usb_device($d->{host});
2909             if (defined($d->{vendorid}) && defined($d->{productid})) {
2910                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2911             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2912                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2913             } elsif (defined($d->{spice}) && $d->{spice}) {
2914                 # usb redir support for spice, currently no usb3
2915                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2916                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2917             }
2918         }
2919     }
2920
2921     # serial devices
2922     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2923         if (my $path = $conf->{"serial$i"}) {
2924             if ($path eq 'socket') {
2925                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2926                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2927                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2928             } else {
2929                 die "no such serial device\n" if ! -c $path;
2930                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2931                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2932             }
2933         }
2934     }
2935
2936     # parallel devices
2937     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2938         if (my $path = $conf->{"parallel$i"}) {
2939             die "no such parallel device\n" if ! -c $path;
2940             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2941             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2942             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2943         }
2944     }
2945
2946     my $vmname = $conf->{name} || "vm$vmid";
2947
2948     push @$cmd, '-name', $vmname;
2949
2950     my $sockets = 1;
2951     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2952     $sockets = $conf->{sockets} if  $conf->{sockets};
2953
2954     my $cores = $conf->{cores} || 1;
2955
2956     my $maxcpus = $sockets * $cores;
2957
2958     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2959
2960     my $allowed_vcpus = $cpuinfo->{cpus};
2961
2962     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2963         if ($allowed_vcpus < $maxcpus);
2964
2965     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2966
2967     push @$cmd, '-nodefaults';
2968
2969     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2970
2971     my $bootindex_hash = {};
2972     my $i = 1;
2973     foreach my $o (split(//, $bootorder)) {
2974         $bootindex_hash->{$o} = $i*100;
2975         $i++;
2976     }
2977
2978     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2979
2980     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2981
2982     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2983
2984     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2985
2986     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2987         my $socket = vnc_socket($vmid);
2988         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2989     } else {
2990         push @$cmd, '-nographic';
2991     }
2992
2993     # time drift fix
2994     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2995
2996     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2997     my $useLocaltime = $conf->{localtime};
2998
2999     if ($ostype) {
3000         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3001
3002         if ($ostype =~ m/^w/) { # windows
3003             $useLocaltime = 1 if !defined($conf->{localtime});
3004
3005             # use time drift fix when acpi is enabled
3006             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3007                 $tdf = 1 if !defined($conf->{tdf});
3008             }
3009         }
3010
3011         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3012             $ostype eq 'wvista') {
3013             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3014             push @$cmd, '-no-hpet';
3015             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3016                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3017                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3018                 push @$cpuFlags , 'hv_time' if !$nokvm;
3019
3020             } else {
3021                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3022             }
3023         }
3024
3025         if ($ostype eq 'win7' || $ostype eq 'win8') {
3026             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3027         }
3028     }
3029
3030     push @$rtcFlags, 'driftfix=slew' if $tdf;
3031
3032     if ($nokvm) {
3033         push @$machineFlags, 'accel=tcg';
3034     } else {
3035         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3036     }
3037
3038     if ($machine_type) {
3039         push @$machineFlags, "type=${machine_type}";
3040     }
3041
3042     if ($conf->{startdate}) {
3043         push @$rtcFlags, "base=$conf->{startdate}";
3044     } elsif ($useLocaltime) {
3045         push @$rtcFlags, 'base=localtime';
3046     }
3047
3048     my $cpu = $nokvm ? "qemu64" : "kvm64";
3049     if (my $cputype = $conf->{cpu}) {
3050         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3051             or die "Cannot parse cpu description: $cputype\n";
3052         $cpu = $cpuconf->{cputype};
3053         $kvm_off = 1 if $cpuconf->{hidden};
3054     }
3055
3056     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3057
3058     push @$cpuFlags , '-x2apic'
3059         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3060
3061     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3062
3063     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3064
3065     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3066
3067         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3068         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3069     }
3070
3071     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3072
3073     push @$cpuFlags, 'kvm=off' if $kvm_off;
3074
3075     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3076
3077     push @$cmd, '-cpu', $cpu;
3078
3079     my $memory = $conf->{memory} || $defaults->{memory};
3080     my $static_memory = 0;
3081     my $dimm_memory = 0;
3082
3083     if ($hotplug_features->{memory}) {
3084         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3085         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3086         $static_memory = $STATICMEM;
3087         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3088         $dimm_memory = $memory - $static_memory;
3089         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3090
3091     } else {
3092
3093         $static_memory = $memory;
3094         push @$cmd, '-m', $static_memory;
3095     }
3096
3097     if ($conf->{numa}) {
3098
3099         my $numa_totalmemory = undef;
3100         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3101             next if !$conf->{"numa$i"};
3102             my $numa = parse_numa($conf->{"numa$i"});
3103             next if !$numa;
3104             # memory
3105             die "missing numa node$i memory value\n" if !$numa->{memory};
3106             my $numa_memory = $numa->{memory};
3107             $numa_totalmemory += $numa_memory;
3108             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3109
3110             # cpus
3111             my $cpulists = $numa->{cpus};
3112             die "missing numa node$i cpus\n" if !defined($cpulists);
3113             my $cpus = join(',', map {
3114                 my ($start, $end) = @$_;
3115                 defined($end) ? "$start-$end" : $start
3116             } @$cpulists);
3117
3118             # hostnodes
3119             my $hostnodelists = $numa->{hostnodes};
3120             if (defined($hostnodelists)) {
3121                 my $hostnodes;
3122                 foreach my $hostnoderange (@$hostnodelists) {
3123                     my ($start, $end) = @$hostnoderange;
3124                     $hostnodes .= ',' if $hostnodes;
3125                     $hostnodes .= $start;
3126                     $hostnodes .= "-$end" if defined($end);
3127                     $end //= $start;
3128                     for (my $i = $start; $i <= $end; ++$i ) {
3129                         die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3130                     }
3131                 }
3132
3133                 # policy
3134                 my $policy = $numa->{policy};
3135                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3136                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3137             }
3138
3139             push @$cmd, '-object', $numa_object;
3140             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3141         }
3142
3143         die "total memory for NUMA nodes must be equal to vm static memory\n"
3144             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3145
3146         #if no custom tology, we split memory and cores across numa nodes
3147         if(!$numa_totalmemory) {
3148
3149             my $numa_memory = ($static_memory / $sockets) . "M";
3150
3151             for (my $i = 0; $i < $sockets; $i++)  {
3152
3153                 my $cpustart = ($cores * $i);
3154                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3155                 my $cpus = $cpustart;
3156                 $cpus .= "-$cpuend" if $cpuend;
3157
3158                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3159                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3160             }
3161         }
3162     }
3163
3164     if ($hotplug_features->{memory}) {
3165         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3166             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3167             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3168             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3169
3170             #if dimm_memory is not aligned to dimm map
3171             if($current_size > $memory) {
3172                  $conf->{memory} = $current_size;
3173                  PVE::QemuConfig->write_config($vmid, $conf);
3174             }
3175         });
3176     }
3177
3178     push @$cmd, '-S' if $conf->{freeze};
3179
3180     # set keyboard layout
3181     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3182     push @$cmd, '-k', $kb if $kb;
3183
3184     # enable sound
3185     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3186     #push @$cmd, '-soundhw', 'es1370';
3187     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3188
3189     if($conf->{agent}) {
3190         my $qgasocket = qmp_socket($vmid, 1);
3191         my $pciaddr = print_pci_addr("qga0", $bridges);
3192         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3193         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3194         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3195     }
3196
3197     my $spice_port;
3198
3199     if ($qxlnum) {
3200         if ($qxlnum > 1) {
3201             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3202                 for(my $i = 1; $i < $qxlnum; $i++){
3203                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3204                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3205                 }
3206             } else {
3207                 # assume other OS works like Linux
3208                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3209                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3210             }
3211         }
3212
3213         my $pciaddr = print_pci_addr("spice", $bridges);
3214
3215         my $nodename = PVE::INotify::nodename();
3216         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3217         $spice_port = PVE::Tools::next_spice_port($pfamily);
3218
3219         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3220
3221         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3222         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3223         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3224     }
3225
3226     # enable balloon by default, unless explicitly disabled
3227     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3228         $pciaddr = print_pci_addr("balloon0", $bridges);
3229         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3230     }
3231
3232     if ($conf->{watchdog}) {
3233         my $wdopts = parse_watchdog($conf->{watchdog});
3234         $pciaddr = print_pci_addr("watchdog", $bridges);
3235         my $watchdog = $wdopts->{model} || 'i6300esb';
3236         push @$devices, '-device', "$watchdog$pciaddr";
3237         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3238     }
3239
3240     my $vollist = [];
3241     my $scsicontroller = {};
3242     my $ahcicontroller = {};
3243     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3244
3245     # Add iscsi initiator name if available
3246     if (my $initiator = get_initiator_name()) {
3247         push @$devices, '-iscsi', "initiator-name=$initiator";
3248     }
3249
3250     foreach_drive($conf, sub {
3251         my ($ds, $drive) = @_;
3252
3253         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3254             push @$vollist, $drive->{file};
3255         }
3256
3257         $use_virtio = 1 if $ds =~ m/^virtio/;
3258
3259         if (drive_is_cdrom ($drive)) {
3260             if ($bootindex_hash->{d}) {
3261                 $drive->{bootindex} = $bootindex_hash->{d};
3262                 $bootindex_hash->{d} += 1;
3263             }
3264         } else {
3265             if ($bootindex_hash->{c}) {
3266                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3267                 $bootindex_hash->{c} += 1;
3268             }
3269         }
3270
3271         if($drive->{interface} eq 'virtio'){
3272            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3273         }
3274
3275         if ($drive->{interface} eq 'scsi') {
3276
3277             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3278
3279             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3280             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3281
3282             my $iothread = '';
3283             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3284                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3285                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3286             }
3287
3288             my $queues = '';
3289             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3290                 $queues = ",num_queues=$drive->{queues}";
3291             }
3292
3293             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3294             $scsicontroller->{$controller}=1;
3295         }
3296
3297         if ($drive->{interface} eq 'sata') {
3298            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3299            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3300            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3301            $ahcicontroller->{$controller}=1;
3302         }
3303
3304         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3305         push @$devices, '-drive',$drive_cmd;
3306         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3307     });
3308
3309     for (my $i = 0; $i < $MAX_NETS; $i++) {
3310          next if !$conf->{"net$i"};
3311          my $d = parse_net($conf->{"net$i"});
3312          next if !$d;
3313
3314          $use_virtio = 1 if $d->{model} eq 'virtio';
3315
3316          if ($bootindex_hash->{n}) {
3317             $d->{bootindex} = $bootindex_hash->{n};
3318             $bootindex_hash->{n} += 1;
3319          }
3320
3321          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3322          push @$devices, '-netdev', $netdevfull;
3323
3324          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3325          push @$devices, '-device', $netdevicefull;
3326     }
3327
3328     if (!$q35) {
3329         # add pci bridges
3330         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3331            $bridges->{1} = 1;
3332            $bridges->{2} = 1;
3333         }
3334
3335         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3336
3337         while (my ($k, $v) = each %$bridges) {
3338             $pciaddr = print_pci_addr("pci.$k");
3339             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3340         }
3341     }
3342
3343     # add custom args
3344     if ($conf->{args}) {
3345         my $aa = PVE::Tools::split_args($conf->{args});
3346         push @$cmd, @$aa;
3347     }
3348
3349     push @$cmd, @$devices;
3350     push @$cmd, '-rtc', join(',', @$rtcFlags)
3351         if scalar(@$rtcFlags);
3352     push @$cmd, '-machine', join(',', @$machineFlags)
3353         if scalar(@$machineFlags);
3354     push @$cmd, '-global', join(',', @$globalFlags)
3355         if scalar(@$globalFlags);
3356
3357     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3358 }
3359
3360 sub vnc_socket {
3361     my ($vmid) = @_;
3362     return "${var_run_tmpdir}/$vmid.vnc";
3363 }
3364
3365 sub spice_port {
3366     my ($vmid) = @_;
3367
3368     my $res = vm_mon_cmd($vmid, 'query-spice');
3369
3370     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3371 }
3372
3373 sub qmp_socket {
3374     my ($vmid, $qga) = @_;
3375     my $sockettype = $qga ? 'qga' : 'qmp';
3376     return "${var_run_tmpdir}/$vmid.$sockettype";
3377 }
3378
3379 sub pidfile_name {
3380     my ($vmid) = @_;
3381     return "${var_run_tmpdir}/$vmid.pid";
3382 }
3383
3384 sub vm_devices_list {
3385     my ($vmid) = @_;
3386
3387     my $res = vm_mon_cmd($vmid, 'query-pci');
3388     my $devices = {};
3389     foreach my $pcibus (@$res) {
3390         foreach my $device (@{$pcibus->{devices}}) {
3391             next if !$device->{'qdev_id'};
3392             if ($device->{'pci_bridge'}) {
3393                 $devices->{$device->{'qdev_id'}} = 1;
3394                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3395                     next if !$bridge_device->{'qdev_id'};
3396                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3397                     $devices->{$device->{'qdev_id'}}++;
3398                 }
3399             } else {
3400                 $devices->{$device->{'qdev_id'}} = 1;
3401             }
3402         }
3403     }
3404
3405     my $resblock = vm_mon_cmd($vmid, 'query-block');
3406     foreach my $block (@$resblock) {
3407         if($block->{device} =~ m/^drive-(\S+)/){
3408                 $devices->{$1} = 1;
3409         }
3410     }
3411
3412     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3413     foreach my $mice (@$resmice) {
3414         if ($mice->{name} eq 'QEMU HID Tablet') {
3415             $devices->{tablet} = 1;
3416             last;
3417         }
3418     }
3419
3420     return $devices;
3421 }
3422
3423 sub vm_deviceplug {
3424     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3425
3426     my $q35 = machine_type_is_q35($conf);
3427
3428     my $devices_list = vm_devices_list($vmid);
3429     return 1 if defined($devices_list->{$deviceid});
3430
3431     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3432
3433     if ($deviceid eq 'tablet') {
3434
3435         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3436
3437     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3438
3439         qemu_iothread_add($vmid, $deviceid, $device);
3440
3441         qemu_driveadd($storecfg, $vmid, $device);
3442         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3443
3444         qemu_deviceadd($vmid, $devicefull);
3445         eval { qemu_deviceaddverify($vmid, $deviceid); };
3446         if (my $err = $@) {
3447             eval { qemu_drivedel($vmid, $deviceid); };
3448             warn $@ if $@;
3449             die $err;
3450         }
3451
3452     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3453
3454
3455         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3456         my $pciaddr = print_pci_addr($deviceid);
3457         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3458
3459         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3460
3461         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3462             qemu_iothread_add($vmid, $deviceid, $device);
3463             $devicefull .= ",iothread=iothread-$deviceid";
3464         }
3465
3466         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3467             $devicefull .= ",num_queues=$device->{queues}";
3468         }
3469
3470         qemu_deviceadd($vmid, $devicefull);
3471         qemu_deviceaddverify($vmid, $deviceid);
3472
3473     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3474
3475         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3476         qemu_driveadd($storecfg, $vmid, $device);
3477
3478         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3479         eval { qemu_deviceadd($vmid, $devicefull); };
3480         if (my $err = $@) {
3481             eval { qemu_drivedel($vmid, $deviceid); };
3482             warn $@ if $@;
3483             die $err;
3484         }
3485
3486     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3487
3488         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3489
3490         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3491         my $use_old_bios_files = undef;
3492         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3493
3494         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3495         qemu_deviceadd($vmid, $netdevicefull);
3496         eval { qemu_deviceaddverify($vmid, $deviceid); };
3497         if (my $err = $@) {
3498             eval { qemu_netdevdel($vmid, $deviceid); };
3499             warn $@ if $@;
3500             die $err;
3501         }
3502
3503     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3504
3505         my $bridgeid = $2;
3506         my $pciaddr = print_pci_addr($deviceid);
3507         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3508
3509         qemu_deviceadd($vmid, $devicefull);
3510         qemu_deviceaddverify($vmid, $deviceid);
3511
3512     } else {
3513         die "can't hotplug device '$deviceid'\n";
3514     }
3515
3516     return 1;
3517 }
3518
3519 # fixme: this should raise exceptions on error!
3520 sub vm_deviceunplug {
3521     my ($vmid, $conf, $deviceid) = @_;
3522
3523     my $devices_list = vm_devices_list($vmid);
3524     return 1 if !defined($devices_list->{$deviceid});
3525
3526     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3527
3528     if ($deviceid eq 'tablet') {
3529
3530         qemu_devicedel($vmid, $deviceid);
3531
3532     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3533
3534         qemu_devicedel($vmid, $deviceid);
3535         qemu_devicedelverify($vmid, $deviceid);
3536         qemu_drivedel($vmid, $deviceid);
3537         qemu_iothread_del($conf, $vmid, $deviceid);
3538
3539     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3540
3541         qemu_devicedel($vmid, $deviceid);
3542         qemu_devicedelverify($vmid, $deviceid);
3543         qemu_iothread_del($conf, $vmid, $deviceid);
3544
3545     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3546
3547         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3548         my $device = parse_drive($deviceid, $conf->{$deviceid});
3549         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3550
3551         qemu_devicedel($vmid, $deviceid);
3552         qemu_drivedel($vmid, $deviceid);
3553         qemu_deletescsihw($conf, $vmid, $deviceid);
3554
3555     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3556
3557         qemu_devicedel($vmid, $deviceid);
3558         qemu_devicedelverify($vmid, $deviceid);
3559         qemu_netdevdel($vmid, $deviceid);
3560
3561     } else {
3562         die "can't unplug device '$deviceid'\n";
3563     }
3564
3565     return 1;
3566 }
3567
3568 sub qemu_deviceadd {
3569     my ($vmid, $devicefull) = @_;
3570
3571     $devicefull = "driver=".$devicefull;
3572     my %options =  split(/[=,]/, $devicefull);
3573
3574     vm_mon_cmd($vmid, "device_add" , %options);
3575 }
3576
3577 sub qemu_devicedel {
3578     my ($vmid, $deviceid) = @_;
3579
3580     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3581 }
3582
3583 sub qemu_iothread_add {
3584     my($vmid, $deviceid, $device) = @_;
3585
3586     if ($device->{iothread}) {
3587         my $iothreads = vm_iothreads_list($vmid);
3588         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3589     }
3590 }
3591
3592 sub qemu_iothread_del {
3593     my($conf, $vmid, $deviceid) = @_;
3594
3595     my $device = parse_drive($deviceid, $conf->{$deviceid});
3596     if ($device->{iothread}) {
3597         my $iothreads = vm_iothreads_list($vmid);
3598         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3599     }
3600 }
3601
3602 sub qemu_objectadd {
3603     my($vmid, $objectid, $qomtype) = @_;
3604
3605     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3606
3607     return 1;
3608 }
3609
3610 sub qemu_objectdel {
3611     my($vmid, $objectid) = @_;
3612
3613     vm_mon_cmd($vmid, "object-del", id => $objectid);
3614
3615     return 1;
3616 }
3617
3618 sub qemu_driveadd {
3619     my ($storecfg, $vmid, $device) = @_;
3620
3621     my $drive = print_drive_full($storecfg, $vmid, $device);
3622     $drive =~ s/\\/\\\\/g;
3623     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3624
3625     # If the command succeeds qemu prints: "OK"
3626     return 1 if $ret =~ m/OK/s;
3627
3628     die "adding drive failed: $ret\n";
3629 }
3630
3631 sub qemu_drivedel {
3632     my($vmid, $deviceid) = @_;
3633
3634     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3635     $ret =~ s/^\s+//;
3636
3637     return 1 if $ret eq "";
3638
3639     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3640     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3641
3642     die "deleting drive $deviceid failed : $ret\n";
3643 }
3644
3645 sub qemu_deviceaddverify {
3646     my ($vmid, $deviceid) = @_;
3647
3648     for (my $i = 0; $i <= 5; $i++) {
3649          my $devices_list = vm_devices_list($vmid);
3650          return 1 if defined($devices_list->{$deviceid});
3651          sleep 1;
3652     }
3653
3654     die "error on hotplug device '$deviceid'\n";
3655 }
3656
3657
3658 sub qemu_devicedelverify {
3659     my ($vmid, $deviceid) = @_;
3660
3661     # need to verify that the device is correctly removed as device_del
3662     # is async and empty return is not reliable
3663
3664     for (my $i = 0; $i <= 5; $i++) {
3665          my $devices_list = vm_devices_list($vmid);
3666          return 1 if !defined($devices_list->{$deviceid});
3667          sleep 1;
3668     }
3669
3670     die "error on hot-unplugging device '$deviceid'\n";
3671 }
3672
3673 sub qemu_findorcreatescsihw {
3674     my ($storecfg, $conf, $vmid, $device) = @_;
3675
3676     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3677
3678     my $scsihwid="$controller_prefix$controller";
3679     my $devices_list = vm_devices_list($vmid);
3680
3681     if(!defined($devices_list->{$scsihwid})) {
3682         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3683     }
3684
3685     return 1;
3686 }
3687
3688 sub qemu_deletescsihw {
3689     my ($conf, $vmid, $opt) = @_;
3690
3691     my $device = parse_drive($opt, $conf->{$opt});
3692
3693     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3694         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3695         return 1;
3696     }
3697
3698     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3699
3700     my $devices_list = vm_devices_list($vmid);
3701     foreach my $opt (keys %{$devices_list}) {
3702         if (PVE::QemuServer::is_valid_drivename($opt)) {
3703             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3704             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3705                 return 1;
3706             }
3707         }
3708     }
3709
3710     my $scsihwid="scsihw$controller";
3711
3712     vm_deviceunplug($vmid, $conf, $scsihwid);
3713
3714     return 1;
3715 }
3716
3717 sub qemu_add_pci_bridge {
3718     my ($storecfg, $conf, $vmid, $device) = @_;
3719
3720     my $bridges = {};
3721
3722     my $bridgeid;
3723
3724     print_pci_addr($device, $bridges);
3725
3726     while (my ($k, $v) = each %$bridges) {
3727         $bridgeid = $k;
3728     }
3729     return 1 if !defined($bridgeid) || $bridgeid < 1;
3730
3731     my $bridge = "pci.$bridgeid";
3732     my $devices_list = vm_devices_list($vmid);
3733
3734     if (!defined($devices_list->{$bridge})) {
3735         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3736     }
3737
3738     return 1;
3739 }
3740
3741 sub qemu_set_link_status {
3742     my ($vmid, $device, $up) = @_;
3743
3744     vm_mon_cmd($vmid, "set_link", name => $device,
3745                up => $up ? JSON::true : JSON::false);
3746 }
3747
3748 sub qemu_netdevadd {
3749     my ($vmid, $conf, $device, $deviceid) = @_;
3750
3751     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3752     my %options =  split(/[=,]/, $netdev);
3753
3754     vm_mon_cmd($vmid, "netdev_add",  %options);
3755     return 1;
3756 }
3757
3758 sub qemu_netdevdel {
3759     my ($vmid, $deviceid) = @_;
3760
3761     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3762 }
3763
3764 sub qemu_cpu_hotplug {
3765     my ($vmid, $conf, $vcpus) = @_;
3766
3767     my $sockets = 1;
3768     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3769     $sockets = $conf->{sockets} if  $conf->{sockets};
3770     my $cores = $conf->{cores} || 1;
3771     my $maxcpus = $sockets * $cores;
3772
3773     $vcpus = $maxcpus if !$vcpus;
3774
3775     die "you can't add more vcpus than maxcpus\n"
3776         if $vcpus > $maxcpus;
3777
3778     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3779     die "online cpu unplug is not yet possible\n"
3780         if $vcpus < $currentvcpus;
3781
3782     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3783     die "vcpus in running vm is different than configuration\n"
3784         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3785
3786     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3787         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3788     }
3789 }
3790
3791 sub qemu_memory_hotplug {
3792     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3793
3794     return $value if !check_running($vmid);
3795
3796     my $memory = $conf->{memory} || $defaults->{memory};
3797     $value = $defaults->{memory} if !$value;
3798     return $value if $value == $memory;
3799
3800     my $static_memory = $STATICMEM;
3801     my $dimm_memory = $memory - $static_memory;
3802
3803     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3804     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3805
3806
3807     my $sockets = 1;
3808     $sockets = $conf->{sockets} if $conf->{sockets};
3809
3810     if($value > $memory) {
3811
3812         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3813             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3814
3815                 return if $current_size <= $conf->{memory};
3816
3817                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3818                 if (my $err = $@) {
3819                     eval { qemu_objectdel($vmid, "mem-$name"); };
3820                     die $err;
3821                 }
3822
3823                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3824                 if (my $err = $@) {
3825                     eval { qemu_objectdel($vmid, "mem-$name"); };
3826                     die $err;
3827                 }
3828                 #update conf after each succesful module hotplug
3829                 $conf->{memory} = $current_size;
3830                 PVE::QemuConfig->write_config($vmid, $conf);
3831         });
3832
3833     } else {
3834
3835         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3836             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3837
3838                 return if $current_size >= $conf->{memory};
3839                 print "try to unplug memory dimm $name\n";
3840
3841                 my $retry = 0;
3842                 while (1) {
3843                     eval { qemu_devicedel($vmid, $name) };
3844                     sleep 3;
3845                     my $dimm_list = qemu_dimm_list($vmid);
3846                     last if !$dimm_list->{$name};
3847                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3848                     $retry++;
3849                 }
3850
3851                 #update conf after each succesful module unplug
3852                 $conf->{memory} = $current_size;
3853
3854                 eval { qemu_objectdel($vmid, "mem-$name"); };
3855                 PVE::QemuConfig->write_config($vmid, $conf);
3856         });
3857     }
3858 }
3859
3860 sub qemu_dimm_list {
3861     my ($vmid) = @_;
3862
3863     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3864     my $dimms = {};
3865
3866     foreach my $dimm (@$dimmarray) {
3867
3868         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3869         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3870         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3871         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3872         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3873     }
3874     return $dimms;
3875 }
3876
3877 sub qemu_block_set_io_throttle {
3878     my ($vmid, $deviceid,
3879         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3880         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3881
3882     return if !check_running($vmid) ;
3883
3884     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3885         bps => int($bps),
3886         bps_rd => int($bps_rd),
3887         bps_wr => int($bps_wr),
3888         iops => int($iops),
3889         iops_rd => int($iops_rd),
3890         iops_wr => int($iops_wr),
3891         bps_max => int($bps_max),
3892         bps_rd_max => int($bps_rd_max),
3893         bps_wr_max => int($bps_wr_max),
3894         iops_max => int($iops_max),
3895         iops_rd_max => int($iops_rd_max),
3896         iops_wr_max => int($iops_wr_max)
3897     );
3898
3899 }
3900
3901 # old code, only used to shutdown old VM after update
3902 sub __read_avail {
3903     my ($fh, $timeout) = @_;
3904
3905     my $sel = new IO::Select;
3906     $sel->add($fh);
3907
3908     my $res = '';
3909     my $buf;
3910
3911     my @ready;
3912     while (scalar (@ready = $sel->can_read($timeout))) {
3913         my $count;
3914         if ($count = $fh->sysread($buf, 8192)) {
3915             if ($buf =~ /^(.*)\(qemu\) $/s) {
3916                 $res .= $1;
3917                 last;
3918             } else {
3919                 $res .= $buf;
3920             }
3921         } else {
3922             if (!defined($count)) {
3923                 die "$!\n";
3924             }
3925             last;
3926         }
3927     }
3928
3929     die "monitor read timeout\n" if !scalar(@ready);
3930
3931     return $res;
3932 }
3933
3934 # old code, only used to shutdown old VM after update
3935 sub vm_monitor_command {
3936     my ($vmid, $cmdstr, $nocheck) = @_;
3937
3938     my $res;
3939
3940     eval {
3941         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3942
3943         my $sname = "${var_run_tmpdir}/$vmid.mon";
3944
3945         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3946             die "unable to connect to VM $vmid socket - $!\n";
3947
3948         my $timeout = 3;
3949
3950         # hack: migrate sometime blocks the monitor (when migrate_downtime
3951         # is set)
3952         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3953             $timeout = 60*60; # 1 hour
3954         }
3955
3956         # read banner;
3957         my $data = __read_avail($sock, $timeout);
3958
3959         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3960             die "got unexpected qemu monitor banner\n";
3961         }
3962
3963         my $sel = new IO::Select;
3964         $sel->add($sock);
3965
3966         if (!scalar(my @ready = $sel->can_write($timeout))) {
3967             die "monitor write error - timeout";
3968         }
3969
3970         my $fullcmd = "$cmdstr\r";
3971
3972         # syslog('info', "VM $vmid monitor command: $cmdstr");
3973
3974         my $b;
3975         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3976             die "monitor write error - $!";
3977         }
3978
3979         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3980
3981         $timeout = 20;
3982
3983         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3984             $timeout = 60*60; # 1 hour
3985         } elsif ($cmdstr =~ m/^(eject|change)/) {
3986             $timeout = 60; # note: cdrom mount command is slow
3987         }
3988         if ($res = __read_avail($sock, $timeout)) {
3989
3990             my @lines = split("\r?\n", $res);
3991
3992             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3993
3994             $res = join("\n", @lines);
3995             $res .= "\n";
3996         }
3997     };
3998
3999     my $err = $@;
4000
4001     if ($err) {
4002         syslog("err", "VM $vmid monitor command failed - $err");
4003         die $err;
4004     }
4005
4006     return $res;
4007 }
4008
4009 sub qemu_block_resize {
4010     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4011
4012     my $running = check_running($vmid);
4013
4014     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4015
4016     return if !$running;
4017
4018     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4019
4020 }
4021
4022 sub qemu_volume_snapshot {
4023     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4024
4025     my $running = check_running($vmid);
4026
4027     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4028         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4029     } else {
4030         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4031     }
4032 }
4033
4034 sub qemu_volume_snapshot_delete {
4035     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4036
4037     my $running = check_running($vmid);
4038
4039     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4040
4041     return if !$running;
4042
4043     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4044 }
4045
4046 sub set_migration_caps {
4047     my ($vmid) = @_;
4048
4049     my $cap_ref = [];
4050
4051     my $enabled_cap = {
4052         "auto-converge" => 1,
4053         "xbzrle" => 1,
4054         "x-rdma-pin-all" => 0,
4055         "zero-blocks" => 0,
4056         "compress" => 0
4057     };
4058
4059     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4060
4061     for my $supported_capability (@$supported_capabilities) {
4062         push @$cap_ref, {
4063             capability => $supported_capability->{capability},
4064             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4065         };
4066     }
4067
4068     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4069 }
4070
4071 my $fast_plug_option = {
4072     'lock' => 1,
4073     'name' => 1,
4074     'onboot' => 1,
4075     'shares' => 1,
4076     'startup' => 1,
4077     'description' => 1,
4078 };
4079
4080 # hotplug changes in [PENDING]
4081 # $selection hash can be used to only apply specified options, for
4082 # example: { cores => 1 } (only apply changed 'cores')
4083 # $errors ref is used to return error messages
4084 sub vmconfig_hotplug_pending {
4085     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4086
4087     my $defaults = load_defaults();
4088
4089     # commit values which do not have any impact on running VM first
4090     # Note: those option cannot raise errors, we we do not care about
4091     # $selection and always apply them.
4092
4093     my $add_error = sub {
4094         my ($opt, $msg) = @_;
4095         $errors->{$opt} = "hotplug problem - $msg";
4096     };
4097
4098     my $changes = 0;
4099     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4100         if ($fast_plug_option->{$opt}) {
4101             $conf->{$opt} = $conf->{pending}->{$opt};
4102             delete $conf->{pending}->{$opt};
4103             $changes = 1;
4104         }
4105     }
4106
4107     if ($changes) {
4108         PVE::QemuConfig->write_config($vmid, $conf);
4109         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4110     }
4111
4112     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4113
4114     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4115     while (my ($opt, $force) = each %$pending_delete_hash) {
4116         next if $selection && !$selection->{$opt};
4117         eval {
4118             if ($opt eq 'hotplug') {
4119                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4120             } elsif ($opt eq 'tablet') {
4121                 die "skip\n" if !$hotplug_features->{usb};
4122                 if ($defaults->{tablet}) {
4123                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4124                 } else {
4125                     vm_deviceunplug($vmid, $conf, $opt);
4126                 }
4127             } elsif ($opt eq 'vcpus') {
4128                 die "skip\n" if !$hotplug_features->{cpu};
4129                 qemu_cpu_hotplug($vmid, $conf, undef);
4130             } elsif ($opt eq 'balloon') {
4131                 # enable balloon device is not hotpluggable
4132                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4133             } elsif ($fast_plug_option->{$opt}) {
4134                 # do nothing
4135             } elsif ($opt =~ m/^net(\d+)$/) {
4136                 die "skip\n" if !$hotplug_features->{network};
4137                 vm_deviceunplug($vmid, $conf, $opt);
4138             } elsif (is_valid_drivename($opt)) {
4139                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4140                 vm_deviceunplug($vmid, $conf, $opt);
4141                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4142             } elsif ($opt =~ m/^memory$/) {
4143                 die "skip\n" if !$hotplug_features->{memory};
4144                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4145             } elsif ($opt eq 'cpuunits') {
4146                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4147             } elsif ($opt eq 'cpulimit') {
4148                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4149             } else {
4150                 die "skip\n";
4151             }
4152         };
4153         if (my $err = $@) {
4154             &$add_error($opt, $err) if $err ne "skip\n";
4155         } else {
4156             # save new config if hotplug was successful
4157             delete $conf->{$opt};
4158             vmconfig_undelete_pending_option($conf, $opt);
4159             PVE::QemuConfig->write_config($vmid, $conf);
4160             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4161         }
4162     }
4163
4164     foreach my $opt (keys %{$conf->{pending}}) {
4165         next if $selection && !$selection->{$opt};
4166         my $value = $conf->{pending}->{$opt};
4167         eval {
4168             if ($opt eq 'hotplug') {
4169                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4170             } elsif ($opt eq 'tablet') {
4171                 die "skip\n" if !$hotplug_features->{usb};
4172                 if ($value == 1) {
4173                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4174                 } elsif ($value == 0) {
4175                     vm_deviceunplug($vmid, $conf, $opt);
4176                 }
4177             } elsif ($opt eq 'vcpus') {
4178                 die "skip\n" if !$hotplug_features->{cpu};
4179                 qemu_cpu_hotplug($vmid, $conf, $value);
4180             } elsif ($opt eq 'balloon') {
4181                 # enable/disable balloning device is not hotpluggable
4182                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4183                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4184                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4185
4186                 # allow manual ballooning if shares is set to zero
4187                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4188                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4189                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4190                 }
4191             } elsif ($opt =~ m/^net(\d+)$/) {
4192                 # some changes can be done without hotplug
4193                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4194                                     $vmid, $opt, $value);
4195             } elsif (is_valid_drivename($opt)) {
4196                 # some changes can be done without hotplug
4197                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4198                                      $vmid, $opt, $value, 1);
4199             } elsif ($opt =~ m/^memory$/) { #dimms
4200                 die "skip\n" if !$hotplug_features->{memory};
4201                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4202             } elsif ($opt eq 'cpuunits') {
4203                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4204             } elsif ($opt eq 'cpulimit') {
4205                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4206                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4207             } else {
4208                 die "skip\n";  # skip non-hot-pluggable options
4209             }
4210         };
4211         if (my $err = $@) {
4212             &$add_error($opt, $err) if $err ne "skip\n";
4213         } else {
4214             # save new config if hotplug was successful
4215             $conf->{$opt} = $value;
4216             delete $conf->{pending}->{$opt};
4217             PVE::QemuConfig->write_config($vmid, $conf);
4218             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4219         }
4220     }
4221 }
4222
4223 sub try_deallocate_drive {
4224     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4225
4226     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4227         my $volid = $drive->{file};
4228         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4229             my $sid = PVE::Storage::parse_volume_id($volid);
4230             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4231
4232             # check if the disk is really unused
4233             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4234                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4235             PVE::Storage::vdisk_free($storecfg, $volid);
4236             return 1;
4237         } else {
4238             # If vm is not owner of this disk remove from config
4239             return 1;
4240         }
4241     }
4242
4243     return undef;
4244 }
4245
4246 sub vmconfig_delete_or_detach_drive {
4247     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4248
4249     my $drive = parse_drive($opt, $conf->{$opt});
4250
4251     my $rpcenv = PVE::RPCEnvironment::get();
4252     my $authuser = $rpcenv->get_user();
4253
4254     if ($force) {
4255         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4256         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4257     } else {
4258         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4259     }
4260 }
4261
4262 sub vmconfig_apply_pending {
4263     my ($vmid, $conf, $storecfg) = @_;
4264
4265     # cold plug
4266
4267     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4268     while (my ($opt, $force) = each %$pending_delete_hash) {
4269         die "internal error" if $opt =~ m/^unused/;
4270         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4271         if (!defined($conf->{$opt})) {
4272             vmconfig_undelete_pending_option($conf, $opt);
4273             PVE::QemuConfig->write_config($vmid, $conf);
4274         } elsif (is_valid_drivename($opt)) {
4275             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4276             vmconfig_undelete_pending_option($conf, $opt);
4277             delete $conf->{$opt};
4278             PVE::QemuConfig->write_config($vmid, $conf);
4279         } else {
4280             vmconfig_undelete_pending_option($conf, $opt);
4281             delete $conf->{$opt};
4282             PVE::QemuConfig->write_config($vmid, $conf);
4283         }
4284     }
4285
4286     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4287
4288     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4289         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4290
4291         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4292             # skip if nothing changed
4293         } elsif (is_valid_drivename($opt)) {
4294             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4295                 if defined($conf->{$opt});
4296             $conf->{$opt} = $conf->{pending}->{$opt};
4297         } else {
4298             $conf->{$opt} = $conf->{pending}->{$opt};
4299         }
4300
4301         delete $conf->{pending}->{$opt};
4302         PVE::QemuConfig->write_config($vmid, $conf);
4303     }
4304 }
4305
4306 my $safe_num_ne = sub {
4307     my ($a, $b) = @_;
4308
4309     return 0 if !defined($a) && !defined($b);
4310     return 1 if !defined($a);
4311     return 1 if !defined($b);
4312
4313     return $a != $b;
4314 };
4315
4316 my $safe_string_ne = sub {
4317     my ($a, $b) = @_;
4318
4319     return 0 if !defined($a) && !defined($b);
4320     return 1 if !defined($a);
4321     return 1 if !defined($b);
4322
4323     return $a ne $b;
4324 };
4325
4326 sub vmconfig_update_net {
4327     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4328
4329     my $newnet = parse_net($value);
4330
4331     if ($conf->{$opt}) {
4332         my $oldnet = parse_net($conf->{$opt});
4333
4334         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4335             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4336             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4337             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4338
4339             # for non online change, we try to hot-unplug
4340             die "skip\n" if !$hotplug;
4341             vm_deviceunplug($vmid, $conf, $opt);
4342         } else {
4343
4344             die "internal error" if $opt !~ m/net(\d+)/;
4345             my $iface = "tap${vmid}i$1";
4346
4347             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4348                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4349                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4350                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4351                 PVE::Network::tap_unplug($iface);
4352                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4353             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4354                 # Rate can be applied on its own but any change above needs to
4355                 # include the rate in tap_plug since OVS resets everything.
4356                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4357             }
4358
4359             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4360                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4361             }
4362
4363             return 1;
4364         }
4365     }
4366
4367     if ($hotplug) {
4368         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4369     } else {
4370         die "skip\n";
4371     }
4372 }
4373
4374 sub vmconfig_update_disk {
4375     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4376
4377     # fixme: do we need force?
4378
4379     my $drive = parse_drive($opt, $value);
4380
4381     if ($conf->{$opt}) {
4382
4383         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4384
4385             my $media = $drive->{media} || 'disk';
4386             my $oldmedia = $old_drive->{media} || 'disk';
4387             die "unable to change media type\n" if $media ne $oldmedia;
4388
4389             if (!drive_is_cdrom($old_drive)) {
4390
4391                 if ($drive->{file} ne $old_drive->{file}) {
4392
4393                     die "skip\n" if !$hotplug;
4394
4395                     # unplug and register as unused
4396                     vm_deviceunplug($vmid, $conf, $opt);
4397                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4398
4399                 } else {
4400                     # update existing disk
4401
4402                     # skip non hotpluggable value
4403                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4404                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4405                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4406                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4407                         die "skip\n";
4408                     }
4409
4410                     # apply throttle
4411                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4412                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4413                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4414                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4415                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4416                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4417                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4418                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4419                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4420                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4421                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4422                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4423
4424                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4425                                                    ($drive->{mbps} || 0)*1024*1024,
4426                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4427                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4428                                                    $drive->{iops} || 0,
4429                                                    $drive->{iops_rd} || 0,
4430                                                    $drive->{iops_wr} || 0,
4431                                                    ($drive->{mbps_max} || 0)*1024*1024,
4432                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4433                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4434                                                    $drive->{iops_max} || 0,
4435                                                    $drive->{iops_rd_max} || 0,
4436                                                    $drive->{iops_wr_max} || 0);
4437
4438                     }
4439
4440                     return 1;
4441                 }
4442
4443             } else { # cdrom
4444
4445                 if ($drive->{file} eq 'none') {
4446                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4447                 } else {
4448                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4449                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4450                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4451                 }
4452
4453                 return 1;
4454             }
4455         }
4456     }
4457
4458     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4459     # hotplug new disks
4460     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4461     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4462 }
4463
4464 sub vm_start {
4465     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4466         $forcemachine, $spice_ticket) = @_;
4467
4468     PVE::QemuConfig->lock_config($vmid, sub {
4469         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4470
4471         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4472
4473         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4474
4475         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4476
4477         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4478             vmconfig_apply_pending($vmid, $conf, $storecfg);
4479             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4480         }
4481
4482         my $defaults = load_defaults();
4483
4484         # set environment variable useful inside network script
4485         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4486
4487         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4488
4489         my $migrate_port = 0;
4490         my $migrate_uri;
4491         if ($statefile) {
4492             if ($statefile eq 'tcp') {
4493                 my $localip = "localhost";
4494                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4495                 my $nodename = PVE::INotify::nodename();
4496                 if ($datacenterconf->{migration_unsecure}) {
4497                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4498                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4499                 }
4500                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4501                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4502                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4503                 push @$cmd, '-incoming', $migrate_uri;
4504                 push @$cmd, '-S';
4505             } else {
4506                 push @$cmd, '-loadstate', $statefile;
4507             }
4508         } elsif ($paused) {
4509             push @$cmd, '-S';
4510         }
4511
4512         # host pci devices
4513         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4514           my $d = parse_hostpci($conf->{"hostpci$i"});
4515           next if !$d;
4516           my $pcidevices = $d->{pciid};
4517           foreach my $pcidevice (@$pcidevices) {
4518                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4519
4520                 my $info = pci_device_info("0000:$pciid");
4521                 die "IOMMU not present\n" if !check_iommu_support();
4522                 die "no pci device info for device '$pciid'\n" if !$info;
4523                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4524                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4525           }
4526         }
4527
4528         PVE::Storage::activate_volumes($storecfg, $vollist);
4529
4530         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4531                     umask => 0077); };
4532
4533         if (my $err = $@) {
4534             # deactivate volumes if start fails
4535             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4536             die "start failed: $err";
4537         }
4538
4539         print "migration listens on $migrate_uri\n" if $migrate_uri;
4540
4541         if ($statefile && $statefile ne 'tcp')  {
4542             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4543             warn $@ if $@;
4544         }
4545
4546         if ($migratedfrom) {
4547
4548             eval {
4549                 set_migration_caps($vmid);
4550             };
4551             warn $@ if $@;
4552
4553             if ($spice_port) {
4554                 print "spice listens on port $spice_port\n";
4555                 if ($spice_ticket) {
4556                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4557                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4558                 }
4559             }
4560
4561         } else {
4562
4563             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4564                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4565                     if $conf->{balloon};
4566             }
4567
4568             foreach my $opt (keys %$conf) {
4569                 next if $opt !~  m/^net\d+$/;
4570                 my $nicconf = parse_net($conf->{$opt});
4571                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4572             }
4573         }
4574
4575         vm_mon_cmd_nocheck($vmid, 'qom-set',
4576                     path => "machine/peripheral/balloon0",
4577                     property => "guest-stats-polling-interval",
4578                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4579
4580     });
4581 }
4582
4583 sub vm_mon_cmd {
4584     my ($vmid, $execute, %params) = @_;
4585
4586     my $cmd = { execute => $execute, arguments => \%params };
4587     vm_qmp_command($vmid, $cmd);
4588 }
4589
4590 sub vm_mon_cmd_nocheck {
4591     my ($vmid, $execute, %params) = @_;
4592
4593     my $cmd = { execute => $execute, arguments => \%params };
4594     vm_qmp_command($vmid, $cmd, 1);
4595 }
4596
4597 sub vm_qmp_command {
4598     my ($vmid, $cmd, $nocheck) = @_;
4599
4600     my $res;
4601
4602     my $timeout;
4603     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4604         $timeout = $cmd->{arguments}->{timeout};
4605         delete $cmd->{arguments}->{timeout};
4606     }
4607
4608     eval {
4609         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4610         my $sname = qmp_socket($vmid);
4611         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4612             my $qmpclient = PVE::QMPClient->new();
4613
4614             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4615         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4616             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4617                 if scalar(%{$cmd->{arguments}});
4618             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4619         } else {
4620             die "unable to open monitor socket\n";
4621         }
4622     };
4623     if (my $err = $@) {
4624         syslog("err", "VM $vmid qmp command failed - $err");
4625         die $err;
4626     }
4627
4628     return $res;
4629 }
4630
4631 sub vm_human_monitor_command {
4632     my ($vmid, $cmdline) = @_;
4633
4634     my $res;
4635
4636     my $cmd = {
4637         execute => 'human-monitor-command',
4638         arguments => { 'command-line' => $cmdline},
4639     };
4640
4641     return vm_qmp_command($vmid, $cmd);
4642 }
4643
4644 sub vm_commandline {
4645     my ($storecfg, $vmid) = @_;
4646
4647     my $conf = PVE::QemuConfig->load_config($vmid);
4648
4649     my $defaults = load_defaults();
4650
4651     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4652
4653     return join(' ', @$cmd);
4654 }
4655
4656 sub vm_reset {
4657     my ($vmid, $skiplock) = @_;
4658
4659     PVE::QemuConfig->lock_config($vmid, sub {
4660
4661         my $conf = PVE::QemuConfig->load_config($vmid);
4662
4663         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4664
4665         vm_mon_cmd($vmid, "system_reset");
4666     });
4667 }
4668
4669 sub get_vm_volumes {
4670     my ($conf) = @_;
4671
4672     my $vollist = [];
4673     foreach_volid($conf, sub {
4674         my ($volid, $is_cdrom) = @_;
4675
4676         return if $volid =~ m|^/|;
4677
4678         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4679         return if !$sid;
4680
4681         push @$vollist, $volid;
4682     });
4683
4684     return $vollist;
4685 }
4686
4687 sub vm_stop_cleanup {
4688     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4689
4690     eval {
4691
4692         if (!$keepActive) {
4693             my $vollist = get_vm_volumes($conf);
4694             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4695         }
4696
4697         foreach my $ext (qw(mon qmp pid vnc qga)) {
4698             unlink "/var/run/qemu-server/${vmid}.$ext";
4699         }
4700
4701         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4702     };
4703     warn $@ if $@; # avoid errors - just warn
4704 }
4705
4706 # Note: use $nockeck to skip tests if VM configuration file exists.
4707 # We need that when migration VMs to other nodes (files already moved)
4708 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4709 sub vm_stop {
4710     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4711
4712     $force = 1 if !defined($force) && !$shutdown;
4713
4714     if ($migratedfrom){
4715         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4716         kill 15, $pid if $pid;
4717         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4718         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4719         return;
4720     }
4721
4722     PVE::QemuConfig->lock_config($vmid, sub {
4723
4724         my $pid = check_running($vmid, $nocheck);
4725         return if !$pid;
4726
4727         my $conf;
4728         if (!$nocheck) {
4729             $conf = PVE::QemuConfig->load_config($vmid);
4730             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4731             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4732                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4733                 $timeout = $opts->{down} if $opts->{down};
4734             }
4735         }
4736
4737         $timeout = 60 if !defined($timeout);
4738
4739         eval {
4740             if ($shutdown) {
4741                 if (defined($conf) && $conf->{agent}) {
4742                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4743                 } else {
4744                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4745                 }
4746             } else {
4747                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4748             }
4749         };
4750         my $err = $@;
4751
4752         if (!$err) {
4753             my $count = 0;
4754             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4755                 $count++;
4756                 sleep 1;
4757             }
4758
4759             if ($count >= $timeout) {
4760                 if ($force) {
4761                     warn "VM still running - terminating now with SIGTERM\n";
4762                     kill 15, $pid;
4763                 } else {
4764                     die "VM quit/powerdown failed - got timeout\n";
4765                 }
4766             } else {
4767                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4768                 return;
4769             }
4770         } else {
4771             if ($force) {
4772                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4773                 kill 15, $pid;
4774             } else {
4775                 die "VM quit/powerdown failed\n";
4776             }
4777         }
4778
4779         # wait again
4780         $timeout = 10;
4781
4782         my $count = 0;
4783         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4784             $count++;
4785             sleep 1;
4786         }
4787
4788         if ($count >= $timeout) {
4789             warn "VM still running - terminating now with SIGKILL\n";
4790             kill 9, $pid;
4791             sleep 1;
4792         }
4793
4794         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4795    });
4796 }
4797
4798 sub vm_suspend {
4799     my ($vmid, $skiplock) = @_;
4800
4801     PVE::QemuConfig->lock_config($vmid, sub {
4802
4803         my $conf = PVE::QemuConfig->load_config($vmid);
4804
4805         PVE::QemuConfig->check_lock($conf)
4806             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4807
4808         vm_mon_cmd($vmid, "stop");
4809     });
4810 }
4811
4812 sub vm_resume {
4813     my ($vmid, $skiplock, $nocheck) = @_;
4814
4815     PVE::QemuConfig->lock_config($vmid, sub {
4816
4817         if (!$nocheck) {
4818
4819             my $conf = PVE::QemuConfig->load_config($vmid);
4820
4821             PVE::QemuConfig->check_lock($conf)
4822                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4823
4824             vm_mon_cmd($vmid, "cont");
4825
4826         } else {
4827             vm_mon_cmd_nocheck($vmid, "cont");
4828         }
4829     });
4830 }
4831
4832 sub vm_sendkey {
4833     my ($vmid, $skiplock, $key) = @_;
4834
4835     PVE::QemuConfig->lock_config($vmid, sub {
4836
4837         my $conf = PVE::QemuConfig->load_config($vmid);
4838
4839         # there is no qmp command, so we use the human monitor command
4840         vm_human_monitor_command($vmid, "sendkey $key");
4841     });
4842 }
4843
4844 sub vm_destroy {
4845     my ($storecfg, $vmid, $skiplock) = @_;
4846
4847     PVE::QemuConfig->lock_config($vmid, sub {
4848
4849         my $conf = PVE::QemuConfig->load_config($vmid);
4850
4851         if (!check_running($vmid)) {
4852             destroy_vm($storecfg, $vmid, undef, $skiplock);
4853         } else {
4854             die "VM $vmid is running - destroy failed\n";
4855         }
4856     });
4857 }
4858
4859 # pci helpers
4860
4861 sub file_write {
4862     my ($filename, $buf) = @_;
4863
4864     my $fh = IO::File->new($filename, "w");
4865     return undef if !$fh;
4866
4867     my $res = print $fh $buf;
4868
4869     $fh->close();
4870
4871     return $res;
4872 }
4873
4874 sub pci_device_info {
4875     my ($name) = @_;
4876
4877     my $res;
4878
4879     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4880     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4881
4882     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4883     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4884
4885     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4886     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4887
4888     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4889     return undef if !defined($product) || $product !~ s/^0x//;
4890
4891     $res = {
4892         name => $name,
4893         vendor => $vendor,
4894         product => $product,
4895         domain => $domain,
4896         bus => $bus,
4897         slot => $slot,
4898         func => $func,
4899         irq => $irq,
4900         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4901     };
4902
4903     return $res;
4904 }
4905
4906 sub pci_dev_reset {
4907     my ($dev) = @_;
4908
4909     my $name = $dev->{name};
4910
4911     my $fn = "$pcisysfs/devices/$name/reset";
4912
4913     return file_write($fn, "1");
4914 }
4915
4916 sub pci_dev_bind_to_vfio {
4917     my ($dev) = @_;
4918
4919     my $name = $dev->{name};
4920
4921     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4922
4923     if (!-d $vfio_basedir) {
4924         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4925     }
4926     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4927
4928     my $testdir = "$vfio_basedir/$name";
4929     return 1 if -d $testdir;
4930
4931     my $data = "$dev->{vendor} $dev->{product}";
4932     return undef if !file_write("$vfio_basedir/new_id", $data);
4933
4934     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4935     if (!file_write($fn, $name)) {
4936         return undef if -f $fn;
4937     }
4938
4939     $fn = "$vfio_basedir/bind";
4940     if (! -d $testdir) {
4941         return undef if !file_write($fn, $name);
4942     }
4943
4944     return -d $testdir;
4945 }
4946
4947 sub pci_dev_group_bind_to_vfio {
4948     my ($pciid) = @_;
4949
4950     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4951
4952     if (!-d $vfio_basedir) {
4953         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4954     }
4955     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4956
4957     # get IOMMU group devices
4958     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4959       my @devs = grep /^0000:/, readdir($D);
4960     closedir($D);
4961
4962     foreach my $pciid (@devs) {
4963         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4964
4965         # pci bridges, switches or root ports are not supported
4966         # they have a pci_bus subdirectory so skip them
4967         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4968
4969         my $info = pci_device_info($1);
4970         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4971     }
4972
4973     return 1;
4974 }
4975
4976 sub print_pci_addr {
4977     my ($id, $bridges) = @_;
4978
4979     my $res = '';
4980     my $devices = {
4981         piix3 => { bus => 0, addr => 1 },
4982         #addr2 : first videocard
4983         balloon0 => { bus => 0, addr => 3 },
4984         watchdog => { bus => 0, addr => 4 },
4985         scsihw0 => { bus => 0, addr => 5 },
4986         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4987         scsihw1 => { bus => 0, addr => 6 },
4988         ahci0 => { bus => 0, addr => 7 },
4989         qga0 => { bus => 0, addr => 8 },
4990         spice => { bus => 0, addr => 9 },
4991         virtio0 => { bus => 0, addr => 10 },
4992         virtio1 => { bus => 0, addr => 11 },
4993         virtio2 => { bus => 0, addr => 12 },
4994         virtio3 => { bus => 0, addr => 13 },
4995         virtio4 => { bus => 0, addr => 14 },
4996         virtio5 => { bus => 0, addr => 15 },
4997         hostpci0 => { bus => 0, addr => 16 },
4998         hostpci1 => { bus => 0, addr => 17 },
4999         net0 => { bus => 0, addr => 18 },
5000         net1 => { bus => 0, addr => 19 },
5001         net2 => { bus => 0, addr => 20 },
5002         net3 => { bus => 0, addr => 21 },
5003         net4 => { bus => 0, addr => 22 },
5004         net5 => { bus => 0, addr => 23 },
5005         vga1 => { bus => 0, addr => 24 },
5006         vga2 => { bus => 0, addr => 25 },
5007         vga3 => { bus => 0, addr => 26 },
5008         hostpci2 => { bus => 0, addr => 27 },
5009         hostpci3 => { bus => 0, addr => 28 },
5010         #addr29 : usb-host (pve-usb.cfg)
5011         'pci.1' => { bus => 0, addr => 30 },
5012         'pci.2' => { bus => 0, addr => 31 },
5013         'net6' => { bus => 1, addr => 1 },
5014         'net7' => { bus => 1, addr => 2 },
5015         'net8' => { bus => 1, addr => 3 },
5016         'net9' => { bus => 1, addr => 4 },
5017         'net10' => { bus => 1, addr => 5 },
5018         'net11' => { bus => 1, addr => 6 },
5019         'net12' => { bus => 1, addr => 7 },
5020         'net13' => { bus => 1, addr => 8 },
5021         'net14' => { bus => 1, addr => 9 },
5022         'net15' => { bus => 1, addr => 10 },
5023         'net16' => { bus => 1, addr => 11 },
5024         'net17' => { bus => 1, addr => 12 },
5025         'net18' => { bus => 1, addr => 13 },
5026         'net19' => { bus => 1, addr => 14 },
5027         'net20' => { bus => 1, addr => 15 },
5028         'net21' => { bus => 1, addr => 16 },
5029         'net22' => { bus => 1, addr => 17 },
5030         'net23' => { bus => 1, addr => 18 },
5031         'net24' => { bus => 1, addr => 19 },
5032         'net25' => { bus => 1, addr => 20 },
5033         'net26' => { bus => 1, addr => 21 },
5034         'net27' => { bus => 1, addr => 22 },
5035         'net28' => { bus => 1, addr => 23 },
5036         'net29' => { bus => 1, addr => 24 },
5037         'net30' => { bus => 1, addr => 25 },
5038         'net31' => { bus => 1, addr => 26 },
5039         'xhci' => { bus => 1, addr => 27 },
5040         'virtio6' => { bus => 2, addr => 1 },
5041         'virtio7' => { bus => 2, addr => 2 },
5042         'virtio8' => { bus => 2, addr => 3 },
5043         'virtio9' => { bus => 2, addr => 4 },
5044         'virtio10' => { bus => 2, addr => 5 },
5045         'virtio11' => { bus => 2, addr => 6 },
5046         'virtio12' => { bus => 2, addr => 7 },
5047         'virtio13' => { bus => 2, addr => 8 },
5048         'virtio14' => { bus => 2, addr => 9 },
5049         'virtio15' => { bus => 2, addr => 10 },
5050         'virtioscsi0' => { bus => 3, addr => 1 },
5051         'virtioscsi1' => { bus => 3, addr => 2 },
5052         'virtioscsi2' => { bus => 3, addr => 3 },
5053         'virtioscsi3' => { bus => 3, addr => 4 },
5054         'virtioscsi4' => { bus => 3, addr => 5 },
5055         'virtioscsi5' => { bus => 3, addr => 6 },
5056         'virtioscsi6' => { bus => 3, addr => 7 },
5057         'virtioscsi7' => { bus => 3, addr => 8 },
5058         'virtioscsi8' => { bus => 3, addr => 9 },
5059         'virtioscsi9' => { bus => 3, addr => 10 },
5060         'virtioscsi10' => { bus => 3, addr => 11 },
5061         'virtioscsi11' => { bus => 3, addr => 12 },
5062         'virtioscsi12' => { bus => 3, addr => 13 },
5063         'virtioscsi13' => { bus => 3, addr => 14 },
5064         'virtioscsi14' => { bus => 3, addr => 15 },
5065         'virtioscsi15' => { bus => 3, addr => 16 },
5066         'virtioscsi16' => { bus => 3, addr => 17 },
5067         'virtioscsi17' => { bus => 3, addr => 18 },
5068         'virtioscsi18' => { bus => 3, addr => 19 },
5069         'virtioscsi19' => { bus => 3, addr => 20 },
5070         'virtioscsi20' => { bus => 3, addr => 21 },
5071         'virtioscsi21' => { bus => 3, addr => 22 },
5072         'virtioscsi22' => { bus => 3, addr => 23 },
5073         'virtioscsi23' => { bus => 3, addr => 24 },
5074         'virtioscsi24' => { bus => 3, addr => 25 },
5075         'virtioscsi25' => { bus => 3, addr => 26 },
5076         'virtioscsi26' => { bus => 3, addr => 27 },
5077         'virtioscsi27' => { bus => 3, addr => 28 },
5078         'virtioscsi28' => { bus => 3, addr => 29 },
5079         'virtioscsi29' => { bus => 3, addr => 30 },
5080         'virtioscsi30' => { bus => 3, addr => 31 },
5081
5082     };
5083
5084     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5085            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5086            my $bus = $devices->{$id}->{bus};
5087            $res = ",bus=pci.$bus,addr=$addr";
5088            $bridges->{$bus} = 1 if $bridges;
5089     }
5090     return $res;
5091
5092 }
5093
5094 sub print_pcie_addr {
5095     my ($id) = @_;
5096
5097     my $res = '';
5098     my $devices = {
5099         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5100         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5101         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5102         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5103     };
5104
5105     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5106            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5107            my $bus = $devices->{$id}->{bus};
5108            $res = ",bus=$bus,addr=$addr";
5109     }
5110     return $res;
5111
5112 }
5113
5114 # vzdump restore implementaion
5115
5116 sub tar_archive_read_firstfile {
5117     my $archive = shift;
5118
5119     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5120
5121     # try to detect archive type first
5122     my $pid = open (TMP, "tar tf '$archive'|") ||
5123         die "unable to open file '$archive'\n";
5124     my $firstfile = <TMP>;
5125     kill 15, $pid;
5126     close TMP;
5127
5128     die "ERROR: archive contaions no data\n" if !$firstfile;
5129     chomp $firstfile;
5130
5131     return $firstfile;
5132 }
5133
5134 sub tar_restore_cleanup {
5135     my ($storecfg, $statfile) = @_;
5136
5137     print STDERR "starting cleanup\n";
5138
5139     if (my $fd = IO::File->new($statfile, "r")) {
5140         while (defined(my $line = <$fd>)) {
5141             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5142                 my $volid = $2;
5143                 eval {
5144                     if ($volid =~ m|^/|) {
5145                         unlink $volid || die 'unlink failed\n';
5146                     } else {
5147                         PVE::Storage::vdisk_free($storecfg, $volid);
5148                     }
5149                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5150                 };
5151                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5152             } else {
5153                 print STDERR "unable to parse line in statfile - $line";
5154             }
5155         }
5156         $fd->close();
5157     }
5158 }
5159
5160 sub restore_archive {
5161     my ($archive, $vmid, $user, $opts) = @_;
5162
5163     my $format = $opts->{format};
5164     my $comp;
5165
5166     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5167         $format = 'tar' if !$format;
5168         $comp = 'gzip';
5169     } elsif ($archive =~ m/\.tar$/) {
5170         $format = 'tar' if !$format;
5171     } elsif ($archive =~ m/.tar.lzo$/) {
5172         $format = 'tar' if !$format;
5173         $comp = 'lzop';
5174     } elsif ($archive =~ m/\.vma$/) {
5175         $format = 'vma' if !$format;
5176     } elsif ($archive =~ m/\.vma\.gz$/) {
5177         $format = 'vma' if !$format;
5178         $comp = 'gzip';
5179     } elsif ($archive =~ m/\.vma\.lzo$/) {
5180         $format = 'vma' if !$format;
5181         $comp = 'lzop';
5182     } else {
5183         $format = 'vma' if !$format; # default
5184     }
5185
5186     # try to detect archive format
5187     if ($format eq 'tar') {
5188         return restore_tar_archive($archive, $vmid, $user, $opts);
5189     } else {
5190         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5191     }
5192 }
5193
5194 sub restore_update_config_line {
5195     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5196
5197     return if $line =~ m/^\#qmdump\#/;
5198     return if $line =~ m/^\#vzdump\#/;
5199     return if $line =~ m/^lock:/;
5200     return if $line =~ m/^unused\d+:/;
5201     return if $line =~ m/^parent:/;
5202     return if $line =~ m/^template:/; # restored VM is never a template
5203
5204     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5205         # try to convert old 1.X settings
5206         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5207         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5208             my ($model, $macaddr) = split(/\=/, $devconfig);
5209             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5210             my $net = {
5211                 model => $model,
5212                 bridge => "vmbr$ind",
5213                 macaddr => $macaddr,
5214             };
5215             my $netstr = print_net($net);
5216
5217             print $outfd "net$cookie->{netcount}: $netstr\n";
5218             $cookie->{netcount}++;
5219         }
5220     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5221         my ($id, $netstr) = ($1, $2);
5222         my $net = parse_net($netstr);
5223         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5224         $netstr = print_net($net);
5225         print $outfd "$id: $netstr\n";
5226     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5227         my $virtdev = $1;
5228         my $value = $3;
5229         my $di = parse_drive($virtdev, $value);
5230         if (defined($di->{backup}) && !$di->{backup}) {
5231             print $outfd "#$line";
5232         } elsif ($map->{$virtdev}) {
5233             delete $di->{format}; # format can change on restore
5234             $di->{file} = $map->{$virtdev};
5235             $value = print_drive($vmid, $di);
5236             print $outfd "$virtdev: $value\n";
5237         } else {
5238             print $outfd $line;
5239         }
5240     } else {
5241         print $outfd $line;
5242     }
5243 }
5244
5245 sub scan_volids {
5246     my ($cfg, $vmid) = @_;
5247
5248     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5249
5250     my $volid_hash = {};
5251     foreach my $storeid (keys %$info) {
5252         foreach my $item (@{$info->{$storeid}}) {
5253             next if !($item->{volid} && $item->{size});
5254             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5255             $volid_hash->{$item->{volid}} = $item;
5256         }
5257     }
5258
5259     return $volid_hash;
5260 }
5261
5262 sub is_volume_in_use {
5263     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5264
5265     my $path = PVE::Storage::path($storecfg, $volid);
5266
5267     my $scan_config = sub {
5268         my ($cref, $snapname) = @_;
5269
5270         foreach my $key (keys %$cref) {
5271             my $value = $cref->{$key};
5272             if (is_valid_drivename($key)) {
5273                 next if $skip_drive && $key eq $skip_drive;
5274                 my $drive = parse_drive($key, $value);
5275                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5276                 return 1 if $volid eq $drive->{file};
5277                 if ($drive->{file} =~ m!^/!) {
5278                     return 1 if $drive->{file} eq $path;
5279                 } else {
5280                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5281                     next if !$storeid;
5282                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5283                     next if !$scfg;
5284                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5285                 }
5286             }
5287         }
5288
5289         return 0;
5290     };
5291
5292     return 1 if &$scan_config($conf);
5293
5294     undef $skip_drive;
5295
5296     foreach my $snapname (keys %{$conf->{snapshots}}) {
5297         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5298     }
5299
5300     return 0;
5301 }
5302
5303 sub update_disksize {
5304     my ($vmid, $conf, $volid_hash) = @_;
5305
5306     my $changes;
5307
5308     my $used = {};
5309
5310     # Note: it is allowed to define multiple storages with same path (alias), so
5311     # we need to check both 'volid' and real 'path' (two different volid can point
5312     # to the same path).
5313
5314     my $usedpath = {};
5315
5316     # update size info
5317     foreach my $opt (keys %$conf) {
5318         if (is_valid_drivename($opt)) {
5319             my $drive = parse_drive($opt, $conf->{$opt});
5320             my $volid = $drive->{file};
5321             next if !$volid;
5322
5323             $used->{$volid} = 1;
5324             if ($volid_hash->{$volid} &&
5325                 (my $path = $volid_hash->{$volid}->{path})) {
5326                 $usedpath->{$path} = 1;
5327             }
5328
5329             next if drive_is_cdrom($drive);
5330             next if !$volid_hash->{$volid};
5331
5332             $drive->{size} = $volid_hash->{$volid}->{size};
5333             my $new = print_drive($vmid, $drive);
5334             if ($new ne $conf->{$opt}) {
5335                 $changes = 1;
5336                 $conf->{$opt} = $new;
5337             }
5338         }
5339     }
5340
5341     # remove 'unusedX' entry if volume is used
5342     foreach my $opt (keys %$conf) {
5343         next if $opt !~ m/^unused\d+$/;
5344         my $volid = $conf->{$opt};
5345         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5346         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5347             $changes = 1;
5348             delete $conf->{$opt};
5349         }
5350     }
5351
5352     foreach my $volid (sort keys %$volid_hash) {
5353         next if $volid =~ m/vm-$vmid-state-/;
5354         next if $used->{$volid};
5355         my $path = $volid_hash->{$volid}->{path};
5356         next if !$path; # just to be sure
5357         next if $usedpath->{$path};
5358         $changes = 1;
5359         PVE::QemuConfig->add_unused_volume($conf, $volid);
5360         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5361     }
5362
5363     return $changes;
5364 }
5365
5366 sub rescan {
5367     my ($vmid, $nolock) = @_;
5368
5369     my $cfg = PVE::Storage::config();
5370
5371     my $volid_hash = scan_volids($cfg, $vmid);
5372
5373     my $updatefn =  sub {
5374         my ($vmid) = @_;
5375
5376         my $conf = PVE::QemuConfig->load_config($vmid);
5377
5378         PVE::QemuConfig->check_lock($conf);
5379
5380         my $vm_volids = {};
5381         foreach my $volid (keys %$volid_hash) {
5382             my $info = $volid_hash->{$volid};
5383             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5384         }
5385
5386         my $changes = update_disksize($vmid, $conf, $vm_volids);
5387
5388         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5389     };
5390
5391     if (defined($vmid)) {
5392         if ($nolock) {
5393             &$updatefn($vmid);
5394         } else {
5395             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5396         }
5397     } else {
5398         my $vmlist = config_list();
5399         foreach my $vmid (keys %$vmlist) {
5400             if ($nolock) {
5401                 &$updatefn($vmid);
5402             } else {
5403                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5404             }
5405         }
5406     }
5407 }
5408
5409 sub restore_vma_archive {
5410     my ($archive, $vmid, $user, $opts, $comp) = @_;
5411
5412     my $input = $archive eq '-' ? "<&STDIN" : undef;
5413     my $readfrom = $archive;
5414
5415     my $uncomp = '';
5416     if ($comp) {
5417         $readfrom = '-';
5418         my $qarchive = PVE::Tools::shellquote($archive);
5419         if ($comp eq 'gzip') {
5420             $uncomp = "zcat $qarchive|";
5421         } elsif ($comp eq 'lzop') {
5422             $uncomp = "lzop -d -c $qarchive|";
5423         } else {
5424             die "unknown compression method '$comp'\n";
5425         }
5426
5427     }
5428
5429     my $tmpdir = "/var/tmp/vzdumptmp$$";
5430     rmtree $tmpdir;
5431
5432     # disable interrupts (always do cleanups)
5433     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5434         warn "got interrupt - ignored\n";
5435     };
5436
5437     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5438     POSIX::mkfifo($mapfifo, 0600);
5439     my $fifofh;
5440
5441     my $openfifo = sub {
5442         open($fifofh, '>', $mapfifo) || die $!;
5443     };
5444
5445     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5446
5447     my $oldtimeout;
5448     my $timeout = 5;
5449
5450     my $devinfo = {};
5451
5452     my $rpcenv = PVE::RPCEnvironment::get();
5453
5454     my $conffile = PVE::QemuConfig->config_file($vmid);
5455     my $tmpfn = "$conffile.$$.tmp";
5456
5457     # Note: $oldconf is undef if VM does not exists
5458     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5459     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5460
5461     my $print_devmap = sub {
5462         my $virtdev_hash = {};
5463
5464         my $cfgfn = "$tmpdir/qemu-server.conf";
5465
5466         # we can read the config - that is already extracted
5467         my $fh = IO::File->new($cfgfn, "r") ||
5468             "unable to read qemu-server.conf - $!\n";
5469
5470         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5471         if (-f $fwcfgfn) {
5472             my $pve_firewall_dir = '/etc/pve/firewall';
5473             mkdir $pve_firewall_dir; # make sure the dir exists
5474             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5475         }
5476
5477         while (defined(my $line = <$fh>)) {
5478             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5479                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5480                 die "archive does not contain data for drive '$virtdev'\n"
5481                     if !$devinfo->{$devname};
5482                 if (defined($opts->{storage})) {
5483                     $storeid = $opts->{storage} || 'local';
5484                 } elsif (!$storeid) {
5485                     $storeid = 'local';
5486                 }
5487                 $format = 'raw' if !$format;
5488                 $devinfo->{$devname}->{devname} = $devname;
5489                 $devinfo->{$devname}->{virtdev} = $virtdev;
5490                 $devinfo->{$devname}->{format} = $format;
5491                 $devinfo->{$devname}->{storeid} = $storeid;
5492
5493                 # check permission on storage
5494                 my $pool = $opts->{pool}; # todo: do we need that?
5495                 if ($user ne 'root@pam') {
5496                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5497                 }
5498
5499                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5500             }
5501         }
5502
5503         foreach my $devname (keys %$devinfo) {
5504             die "found no device mapping information for device '$devname'\n"
5505                 if !$devinfo->{$devname}->{virtdev};
5506         }
5507
5508         my $cfg = PVE::Storage::config();
5509
5510         # create empty/temp config
5511         if ($oldconf) {
5512             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5513             foreach_drive($oldconf, sub {
5514                 my ($ds, $drive) = @_;
5515
5516                 return if drive_is_cdrom($drive);
5517
5518                 my $volid = $drive->{file};
5519
5520                 return if !$volid || $volid =~ m|^/|;
5521
5522                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5523                 return if !$path || !$owner || ($owner != $vmid);
5524
5525                 # Note: only delete disk we want to restore
5526                 # other volumes will become unused
5527                 if ($virtdev_hash->{$ds}) {
5528                     PVE::Storage::vdisk_free($cfg, $volid);
5529                 }
5530             });
5531
5532             # delete vmstate files
5533             # since after the restore we have no snapshots anymore
5534             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5535                 my $snap = $oldconf->{snapshots}->{$snapname};
5536                 if ($snap->{vmstate}) {
5537                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5538                     if (my $err = $@) {
5539                         warn $err;
5540                     }
5541                 }
5542             }
5543         }
5544
5545         my $map = {};
5546         foreach my $virtdev (sort keys %$virtdev_hash) {
5547             my $d = $virtdev_hash->{$virtdev};
5548             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5549             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5550
5551             # test if requested format is supported
5552             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5553             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5554             $d->{format} = $defFormat if !$supported;
5555
5556             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5557                                                   $d->{format}, undef, $alloc_size);
5558             print STDERR "new volume ID is '$volid'\n";
5559             $d->{volid} = $volid;
5560             my $path = PVE::Storage::path($cfg, $volid);
5561
5562             PVE::Storage::activate_volumes($cfg,[$volid]);
5563
5564             my $write_zeros = 1;
5565             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5566                 $write_zeros = 0;
5567             }
5568
5569             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5570
5571             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5572             $map->{$virtdev} = $volid;
5573         }
5574
5575         $fh->seek(0, 0) || die "seek failed - $!\n";
5576
5577         my $outfd = new IO::File ($tmpfn, "w") ||
5578             die "unable to write config for VM $vmid\n";
5579
5580         my $cookie = { netcount => 0 };
5581         while (defined(my $line = <$fh>)) {
5582             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5583         }
5584
5585         $fh->close();
5586         $outfd->close();
5587     };
5588
5589     eval {
5590         # enable interrupts
5591         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5592             die "interrupted by signal\n";
5593         };
5594         local $SIG{ALRM} = sub { die "got timeout\n"; };
5595
5596         $oldtimeout = alarm($timeout);
5597
5598         my $parser = sub {
5599             my $line = shift;
5600
5601             print "$line\n";
5602
5603             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5604                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5605                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5606             } elsif ($line =~ m/^CTIME: /) {
5607                 # we correctly received the vma config, so we can disable
5608                 # the timeout now for disk allocation (set to 10 minutes, so
5609                 # that we always timeout if something goes wrong)
5610                 alarm(600);
5611                 &$print_devmap();
5612                 print $fifofh "done\n";
5613                 my $tmp = $oldtimeout || 0;
5614                 $oldtimeout = undef;
5615                 alarm($tmp);
5616                 close($fifofh);
5617             }
5618         };
5619
5620         print "restore vma archive: $cmd\n";
5621         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5622     };
5623     my $err = $@;
5624
5625     alarm($oldtimeout) if $oldtimeout;
5626
5627     my $vollist = [];
5628     foreach my $devname (keys %$devinfo) {
5629         my $volid = $devinfo->{$devname}->{volid};
5630         push @$vollist, $volid if $volid;
5631     }
5632
5633     my $cfg = PVE::Storage::config();
5634     PVE::Storage::deactivate_volumes($cfg, $vollist);
5635
5636     unlink $mapfifo;
5637
5638     if ($err) {
5639         rmtree $tmpdir;
5640         unlink $tmpfn;
5641
5642         foreach my $devname (keys %$devinfo) {
5643             my $volid = $devinfo->{$devname}->{volid};
5644             next if !$volid;
5645             eval {
5646                 if ($volid =~ m|^/|) {
5647                     unlink $volid || die 'unlink failed\n';
5648                 } else {
5649                     PVE::Storage::vdisk_free($cfg, $volid);
5650                 }
5651                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5652             };
5653             print STDERR "unable to cleanup '$volid' - $@" if $@;
5654         }
5655         die $err;
5656     }
5657
5658     rmtree $tmpdir;
5659
5660     rename($tmpfn, $conffile) ||
5661         die "unable to commit configuration file '$conffile'\n";
5662
5663     PVE::Cluster::cfs_update(); # make sure we read new file
5664
5665     eval { rescan($vmid, 1); };
5666     warn $@ if $@;
5667 }
5668
5669 sub restore_tar_archive {
5670     my ($archive, $vmid, $user, $opts) = @_;
5671
5672     if ($archive ne '-') {
5673         my $firstfile = tar_archive_read_firstfile($archive);
5674         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5675             if $firstfile ne 'qemu-server.conf';
5676     }
5677
5678     my $storecfg = PVE::Storage::config();
5679
5680     # destroy existing data - keep empty config
5681     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5682     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5683
5684     my $tocmd = "/usr/lib/qemu-server/qmextract";
5685
5686     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5687     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5688     $tocmd .= ' --prealloc' if $opts->{prealloc};
5689     $tocmd .= ' --info' if $opts->{info};
5690
5691     # tar option "xf" does not autodetect compression when read from STDIN,
5692     # so we pipe to zcat
5693     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5694         PVE::Tools::shellquote("--to-command=$tocmd");
5695
5696     my $tmpdir = "/var/tmp/vzdumptmp$$";
5697     mkpath $tmpdir;
5698
5699     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5700     local $ENV{VZDUMP_VMID} = $vmid;
5701     local $ENV{VZDUMP_USER} = $user;
5702
5703     my $conffile = PVE::QemuConfig->config_file($vmid);
5704     my $tmpfn = "$conffile.$$.tmp";
5705
5706     # disable interrupts (always do cleanups)
5707     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5708         print STDERR "got interrupt - ignored\n";
5709     };
5710
5711     eval {
5712         # enable interrupts
5713         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5714             die "interrupted by signal\n";
5715         };
5716
5717         if ($archive eq '-') {
5718             print "extracting archive from STDIN\n";
5719             run_command($cmd, input => "<&STDIN");
5720         } else {
5721             print "extracting archive '$archive'\n";
5722             run_command($cmd);
5723         }
5724
5725         return if $opts->{info};
5726
5727         # read new mapping
5728         my $map = {};
5729         my $statfile = "$tmpdir/qmrestore.stat";
5730         if (my $fd = IO::File->new($statfile, "r")) {
5731             while (defined (my $line = <$fd>)) {
5732                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5733                     $map->{$1} = $2 if $1;
5734                 } else {
5735                     print STDERR "unable to parse line in statfile - $line\n";
5736                 }
5737             }
5738             $fd->close();
5739         }
5740
5741         my $confsrc = "$tmpdir/qemu-server.conf";
5742
5743         my $srcfd = new IO::File($confsrc, "r") ||
5744             die "unable to open file '$confsrc'\n";
5745
5746         my $outfd = new IO::File ($tmpfn, "w") ||
5747             die "unable to write config for VM $vmid\n";
5748
5749         my $cookie = { netcount => 0 };
5750         while (defined (my $line = <$srcfd>)) {
5751             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5752         }
5753
5754         $srcfd->close();
5755         $outfd->close();
5756     };
5757     my $err = $@;
5758
5759     if ($err) {
5760
5761         unlink $tmpfn;
5762
5763         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5764
5765         die $err;
5766     }
5767
5768     rmtree $tmpdir;
5769
5770     rename $tmpfn, $conffile ||
5771         die "unable to commit configuration file '$conffile'\n";
5772
5773     PVE::Cluster::cfs_update(); # make sure we read new file
5774
5775     eval { rescan($vmid, 1); };
5776     warn $@ if $@;
5777 };
5778
5779 sub foreach_writable_storage {
5780     my ($conf, $func) = @_;
5781
5782     my $sidhash = {};
5783
5784     foreach my $ds (keys %$conf) {
5785         next if !is_valid_drivename($ds);
5786
5787         my $drive = parse_drive($ds, $conf->{$ds});
5788         next if !$drive;
5789         next if drive_is_cdrom($drive);
5790
5791         my $volid = $drive->{file};
5792
5793         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5794         $sidhash->{$sid} = $sid if $sid;
5795     }
5796
5797     foreach my $sid (sort keys %$sidhash) {
5798         &$func($sid);
5799     }
5800 }
5801
5802 sub do_snapshots_with_qemu {
5803     my ($storecfg, $volid) = @_;
5804
5805     my $storage_name = PVE::Storage::parse_volume_id($volid);
5806
5807     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5808         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5809         return 1;
5810     }
5811
5812     if ($volid =~ m/\.(qcow2|qed)$/){
5813         return 1;
5814     }
5815
5816     return undef;
5817 }
5818
5819 sub qga_check_running {
5820     my ($vmid) = @_;
5821
5822     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5823     if ($@) {
5824         warn "Qemu Guest Agent are not running - $@";
5825         return 0;
5826     }
5827     return 1;
5828 }
5829
5830 sub template_create {
5831     my ($vmid, $conf, $disk) = @_;
5832
5833     my $storecfg = PVE::Storage::config();
5834
5835     foreach_drive($conf, sub {
5836         my ($ds, $drive) = @_;
5837
5838         return if drive_is_cdrom($drive);
5839         return if $disk && $ds ne $disk;
5840
5841         my $volid = $drive->{file};
5842         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5843
5844         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5845         $drive->{file} = $voliddst;
5846         $conf->{$ds} = print_drive($vmid, $drive);
5847         PVE::QemuConfig->write_config($vmid, $conf);
5848     });
5849 }
5850
5851 sub qemu_img_convert {
5852     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5853
5854     my $storecfg = PVE::Storage::config();
5855     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5856     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5857
5858     if ($src_storeid && $dst_storeid) {
5859
5860         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5861
5862         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5863         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5864
5865         my $src_format = qemu_img_format($src_scfg, $src_volname);
5866         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5867
5868         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5869         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5870
5871         my $cmd = [];
5872         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5873         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5874         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5875         if ($is_zero_initialized) {
5876             push @$cmd, "zeroinit:$dst_path";
5877         } else {
5878             push @$cmd, $dst_path;
5879         }
5880
5881         my $parser = sub {
5882             my $line = shift;
5883             if($line =~ m/\((\S+)\/100\%\)/){
5884                 my $percent = $1;
5885                 my $transferred = int($size * $percent / 100);
5886                 my $remaining = $size - $transferred;
5887
5888                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5889             }
5890
5891         };
5892
5893         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5894         my $err = $@;
5895         die "copy failed: $err" if $err;
5896     }
5897 }
5898
5899 sub qemu_img_format {
5900     my ($scfg, $volname) = @_;
5901
5902     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5903         return $1;
5904     } else {
5905         return "raw";
5906     }
5907 }
5908
5909 sub qemu_drive_mirror {
5910     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5911
5912     my $storecfg = PVE::Storage::config();
5913     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5914
5915     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5916
5917     my $format = qemu_img_format($dst_scfg, $dst_volname);
5918
5919     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5920
5921     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5922
5923     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5924     $opts->{format} = $format if $format;
5925
5926     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5927
5928     my $finish_job = sub {
5929         while (1) {
5930             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5931             my $stat = @$stats[0];
5932             last if !$stat;
5933             sleep 1;
5934         }
5935     };
5936
5937     eval {
5938     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5939         while (1) {
5940             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5941             my $stat = @$stats[0];
5942             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5943             die "error job is not mirroring" if $stat->{type} ne "mirror";
5944
5945             my $busy = $stat->{busy};
5946             my $ready = $stat->{ready};
5947
5948             if (my $total = $stat->{len}) {
5949                 my $transferred = $stat->{offset} || 0;
5950                 my $remaining = $total - $transferred;
5951                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5952
5953                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5954             }
5955
5956
5957             if ($stat->{ready} eq 'true') {
5958
5959                 last if $vmiddst != $vmid;
5960
5961                 # try to switch the disk if source and destination are on the same guest
5962                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5963                 if (!$@) {
5964                     &$finish_job();
5965                     last;
5966                 }
5967                 die $@ if $@ !~ m/cannot be completed/;
5968             }
5969             sleep 1;
5970         }
5971
5972
5973     };
5974     my $err = $@;
5975
5976     my $cancel_job = sub {
5977         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5978         &$finish_job();
5979     };
5980
5981     if ($err) {
5982         eval { &$cancel_job(); };
5983         die "mirroring error: $err";
5984     }
5985
5986     if ($vmiddst != $vmid) {
5987         # if we clone a disk for a new target vm, we don't switch the disk
5988         &$cancel_job(); # so we call block-job-cancel
5989     }
5990 }
5991
5992 sub clone_disk {
5993     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5994         $newvmid, $storage, $format, $full, $newvollist) = @_;
5995
5996     my $newvolid;
5997
5998     if (!$full) {
5999         print "create linked clone of drive $drivename ($drive->{file})\n";
6000         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6001         push @$newvollist, $newvolid;
6002     } else {
6003         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6004         $storeid = $storage if $storage;
6005
6006         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6007         if (!$format) {
6008             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6009             $format = qemu_img_format($scfg, $volname);
6010         }
6011
6012         # test if requested format is supported - else use default
6013         my $supported = grep { $_ eq $format } @$validFormats;
6014         $format = $defFormat if !$supported;
6015
6016         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6017
6018         print "create full clone of drive $drivename ($drive->{file})\n";
6019         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6020         push @$newvollist, $newvolid;
6021
6022         PVE::Storage::activate_volumes($storecfg, $newvollist);
6023
6024         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6025         if (!$running || $snapname) {
6026             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6027         } else {
6028             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
6029         }
6030     }
6031
6032     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6033
6034     my $disk = $drive;
6035     $disk->{format} = undef;
6036     $disk->{file} = $newvolid;
6037     $disk->{size} = $size;
6038
6039     return $disk;
6040 }
6041
6042 # this only works if VM is running
6043 sub get_current_qemu_machine {
6044     my ($vmid) = @_;
6045
6046     my $cmd = { execute => 'query-machines', arguments => {} };
6047     my $res = vm_qmp_command($vmid, $cmd);
6048
6049     my ($current, $default);
6050     foreach my $e (@$res) {
6051         $default = $e->{name} if $e->{'is-default'};
6052         $current = $e->{name} if $e->{'is-current'};
6053     }
6054
6055     # fallback to the default machine if current is not supported by qemu
6056     return $current || $default || 'pc';
6057 }
6058
6059 sub qemu_machine_feature_enabled {
6060     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6061
6062     my $current_major;
6063     my $current_minor;
6064
6065     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6066
6067         $current_major = $3;
6068         $current_minor = $4;
6069
6070     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6071
6072         $current_major = $1;
6073         $current_minor = $2;
6074     }
6075
6076     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6077
6078
6079 }
6080
6081 sub qemu_machine_pxe {
6082     my ($vmid, $conf, $machine) = @_;
6083
6084     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6085
6086     foreach my $opt (keys %$conf) {
6087         next if $opt !~ m/^net(\d+)$/;
6088         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6089         next if !$net;
6090         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6091         return $machine.".pxe" if $romfile =~ m/pxe/;
6092         last;
6093     }
6094
6095     return $machine;
6096 }
6097
6098 sub qemu_use_old_bios_files {
6099     my ($machine_type) = @_;
6100
6101     return if !$machine_type;
6102
6103     my $use_old_bios_files = undef;
6104
6105     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6106         $machine_type = $1;
6107         $use_old_bios_files = 1;
6108     } else {
6109         my $kvmver = kvm_user_version();
6110         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6111         # load new efi bios files on migration. So this hack is required to allow
6112         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6113         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6114         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6115     }
6116
6117     return ($use_old_bios_files, $machine_type);
6118 }
6119
6120 sub lspci {
6121
6122     my $devices = {};
6123
6124     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6125             my (undef, $id, $function) = @_;
6126             my $res = { id => $id, function => $function};
6127             push @{$devices->{$id}}, $res;
6128     });
6129
6130     return $devices;
6131 }
6132
6133 sub vm_iothreads_list {
6134     my ($vmid) = @_;
6135
6136     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6137
6138     my $iothreads = {};
6139     foreach my $iothread (@$res) {
6140         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6141     }
6142
6143     return $iothreads;
6144 }
6145
6146 sub scsihw_infos {
6147     my ($conf, $drive) = @_;
6148
6149     my $maxdev = 0;
6150
6151     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6152         $maxdev = 7;
6153     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6154         $maxdev = 1;
6155     } else {
6156         $maxdev = 256;
6157     }
6158
6159     my $controller = int($drive->{index} / $maxdev);
6160     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6161
6162     return ($maxdev, $controller, $controller_prefix);
6163 }
6164
6165 # bash completion helper
6166
6167 sub complete_backup_archives {
6168     my ($cmdname, $pname, $cvalue) = @_;
6169
6170     my $cfg = PVE::Storage::config();
6171
6172     my $storeid;
6173
6174     if ($cvalue =~ m/^([^:]+):/) {
6175         $storeid = $1;
6176     }
6177
6178     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6179
6180     my $res = [];
6181     foreach my $id (keys %$data) {
6182         foreach my $item (@{$data->{$id}}) {
6183             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6184             push @$res, $item->{volid} if defined($item->{volid});
6185         }
6186     }
6187
6188     return $res;
6189 }
6190
6191 my $complete_vmid_full = sub {
6192     my ($running) = @_;
6193
6194     my $idlist = vmstatus();
6195
6196     my $res = [];
6197
6198     foreach my $id (keys %$idlist) {
6199         my $d = $idlist->{$id};
6200         if (defined($running)) {
6201             next if $d->{template};
6202             next if $running && $d->{status} ne 'running';
6203             next if !$running && $d->{status} eq 'running';
6204         }
6205         push @$res, $id;
6206
6207     }
6208     return $res;
6209 };
6210
6211 sub complete_vmid {
6212     return &$complete_vmid_full();
6213 }
6214
6215 sub complete_vmid_stopped {
6216     return &$complete_vmid_full(0);
6217 }
6218
6219 sub complete_vmid_running {
6220     return &$complete_vmid_full(1);
6221 }
6222
6223 sub complete_storage {
6224
6225     my $cfg = PVE::Storage::config();
6226     my $ids = $cfg->{ids};
6227
6228     my $res = [];
6229     foreach my $sid (keys %$ids) {
6230         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6231         next if !$ids->{$sid}->{content}->{images};
6232         push @$res, $sid;
6233     }
6234
6235     return $res;
6236 }
6237
6238 1;