PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35 use URI::Escape;
  36
  37 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  38
  39 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  40
  41 # Note about locking: we use flock on the config file protect
  42 # against concurent actions.
  43 # Aditionaly, we have a 'lock' setting in the config file. This
  44 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  45 # allowed when such lock is set. But you can ignore this kind of
  46 # lock with the --skiplock flag.
  47
  48 cfs_register_file('/qemu-server/',
  49                   \&parse_vm_config,
  50                   \&write_vm_config);
  51
  52 PVE::JSONSchema::register_standard_option('skiplock', {
  53     description => "Ignore locks - only root is allowed to use this option.",
  54     type => 'boolean',
  55     optional => 1,
  56 });
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 #no warnings 'redefine';
  72
  73 sub cgroups_write {
  74    my ($controller, $vmid, $option, $value) = @_;
  75
  76    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  77    PVE::ProcFSTools::write_proc_entry($path, $value);
  78
  79 }
  80
  81 my $nodename = PVE::INotify::nodename();
  82
  83 mkdir "/etc/pve/nodes/$nodename";
  84 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  85 mkdir $confdir;
  86
  87 my $var_run_tmpdir = "/var/run/qemu-server";
  88 mkdir $var_run_tmpdir;
  89
  90 my $lock_dir = "/var/lock/qemu-server";
  91 mkdir $lock_dir;
  92
  93 my $pcisysfs = "/sys/bus/pci";
  94
  95 my $cpudesc = {
  96     cputype => {
  97         description => "Emulated CPU type.",
  98         type => 'string',
  99         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 100         default => 'kvm64',
 101         default_key => 1,
 102     },
 103     hidden => {
 104         description => "Do not identify as a KVM virtual machine.",
 105         type => 'boolean',
 106         optional => 1,
 107         default => 0
 108     },
 109 };
 110
 111 my $confdesc = {
 112     onboot => {
 113         optional => 1,
 114         type => 'boolean',
 115         description => "Specifies whether a VM will be started during system bootup.",
 116         default => 0,
 117     },
 118     autostart => {
 119         optional => 1,
 120         type => 'boolean',
 121         description => "Automatic restart after crash (currently ignored).",
 122         default => 0,
 123     },
 124     hotplug => {
 125         optional => 1,
 126         type => 'string', format => 'pve-hotplug-features',
 127         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 128         default => 'network,disk,usb',
 129     },
 130     reboot => {
 131         optional => 1,
 132         type => 'boolean',
 133         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 134         default => 1,
 135     },
 136     lock => {
 137         optional => 1,
 138         type => 'string',
 139         description => "Lock/unlock the VM.",
 140         enum => [qw(migrate backup snapshot rollback)],
 141     },
 142     cpulimit => {
 143         optional => 1,
 144         type => 'number',
 145         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 146         minimum => 0,
 147         maximum => 128,
 148         default => 0,
 149     },
 150     cpuunits => {
 151         optional => 1,
 152         type => 'integer',
 153         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 154         minimum => 0,
 155         maximum => 500000,
 156         default => 1000,
 157     },
 158     memory => {
 159         optional => 1,
 160         type => 'integer',
 161         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 162         minimum => 16,
 163         default => 512,
 164     },
 165     balloon => {
 166         optional => 1,
 167         type => 'integer',
 168         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 169         minimum => 0,
 170     },
 171     shares => {
 172         optional => 1,
 173         type => 'integer',
 174         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 175         minimum => 0,
 176         maximum => 50000,
 177         default => 1000,
 178     },
 179     keyboard => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 183         enum => PVE::Tools::kvmkeymaplist(),
 184         default => 'en-us',
 185     },
 186     name => {
 187         optional => 1,
 188         type => 'string', format => 'dns-name',
 189         description => "Set a name for the VM. Only used on the configuration web interface.",
 190     },
 191     scsihw => {
 192         optional => 1,
 193         type => 'string',
 194         description => "scsi controller model",
 195         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 196         default => 'lsi',
 197     },
 198     description => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 202     },
 203     ostype => {
 204         optional => 1,
 205         type => 'string',
 206         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 207         description => <<EODESC,
 208 Used to enable special optimization/features for specific
 209 operating systems:
 210
 211 other  => unspecified OS
 212 wxp    => Microsoft Windows XP
 213 w2k    => Microsoft Windows 2000
 214 w2k3   => Microsoft Windows 2003
 215 w2k8   => Microsoft Windows 2008
 216 wvista => Microsoft Windows Vista
 217 win7   => Microsoft Windows 7
 218 win8   => Microsoft Windows 8/2012
 219 l24    => Linux 2.4 Kernel
 220 l26    => Linux 2.6/3.X Kernel
 221 solaris => solaris/opensolaris/openindiania kernel
 222
 223 other|l24|l26|solaris                       ... no special behaviour
 224 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 225 EODESC
 226     },
 227     boot => {
 228         optional => 1,
 229         type => 'string',
 230         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 231         pattern => '[acdn]{1,4}',
 232         default => 'cdn',
 233     },
 234     bootdisk => {
 235         optional => 1,
 236         type => 'string', format => 'pve-qm-bootdisk',
 237         description => "Enable booting from specified disk.",
 238         pattern => '(ide|sata|scsi|virtio)\d+',
 239     },
 240     smp => {
 241         optional => 1,
 242         type => 'integer',
 243         description => "The number of CPUs. Please use option -sockets instead.",
 244         minimum => 1,
 245         default => 1,
 246     },
 247     sockets => {
 248         optional => 1,
 249         type => 'integer',
 250         description => "The number of CPU sockets.",
 251         minimum => 1,
 252         default => 1,
 253     },
 254     cores => {
 255         optional => 1,
 256         type => 'integer',
 257         description => "The number of cores per socket.",
 258         minimum => 1,
 259         default => 1,
 260     },
 261     numa => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable NUMA.",
 265         default => 0,
 266     },
 267     vcpus => {
 268         optional => 1,
 269         type => 'integer',
 270         description => "Number of hotplugged vcpus.",
 271         minimum => 1,
 272         default => 0,
 273     },
 274     acpi => {
 275         optional => 1,
 276         type => 'boolean',
 277         description => "Enable/disable ACPI.",
 278         default => 1,
 279     },
 280     agent => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Enable/disable Qemu GuestAgent.",
 284         default => 0,
 285     },
 286     kvm => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Enable/disable KVM hardware virtualization.",
 290         default => 1,
 291     },
 292     tdf => {
 293         optional => 1,
 294         type => 'boolean',
 295         description => "Enable/disable time drift fix.",
 296         default => 0,
 297     },
 298     localtime => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 302     },
 303     freeze => {
 304         optional => 1,
 305         type => 'boolean',
 306         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 307     },
 308     vga => {
 309         optional => 1,
 310         type => 'string',
 311         description => "Select the VGA type. If you want to use high resolution" .
 312             " modes (>= 1280x1024x16) then you should use the options " .
 313             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 314             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 315             "display sever. For win* OS you can select how many independent " .
 316             "displays you want, Linux guests can add displays them self. " .
 317             "You can also run without any graphic card, using a serial device" .
 318             " as terminal.",
 319         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 320     },
 321     watchdog => {
 322         optional => 1,
 323         type => 'string', format => 'pve-qm-watchdog',
 324         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 325         description => "Create a virtual hardware watchdog device. Once enabled" .
 326             " (by a guest action), the watchdog must be periodically polled " .
 327             "by an agent inside the guest or else the watchdog will reset " .
 328             "the guest (or execute the respective action specified)",
 329     },
 330     startdate => {
 331         optional => 1,
 332         type => 'string',
 333         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 334         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 335         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 336         default => 'now',
 337     },
 338     startup =>  get_standard_option('pve-startup-order'),
 339     template => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Enable/disable Template.",
 343         default => 0,
 344     },
 345     args => {
 346         optional => 1,
 347         type => 'string',
 348         description => <<EODESCR,
 349 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 350
 351 args: -no-reboot -no-hpet
 352 EODESCR
 353     },
 354     tablet => {
 355         optional => 1,
 356         type => 'boolean',
 357         default => 1,
 358         description => "Enable/disable the USB tablet device. This device is " .
 359             "usually needed to allow absolute mouse positioning with VNC. " .
 360             "Else the mouse runs out of sync with normal VNC clients. " .
 361             "If you're running lots of console-only guests on one host, " .
 362             "you may consider disabling this to save some context switches. " .
 363             "This is turned off by default if you use spice (-vga=qxl).",
 364     },
 365     migrate_speed => {
 366         optional => 1,
 367         type => 'integer',
 368         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 369         minimum => 0,
 370         default => 0,
 371     },
 372     migrate_downtime => {
 373         optional => 1,
 374         type => 'number',
 375         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 376         minimum => 0,
 377         default => 0.1,
 378     },
 379     cdrom => {
 380         optional => 1,
 381         type => 'string', format => 'pve-qm-drive',
 382         typetext => 'volume',
 383         description => "This is an alias for option -ide2",
 384     },
 385     cpu => {
 386         optional => 1,
 387         description => "Emulated CPU type.",
 388         type => 'string',
 389         format => $cpudesc,
 390     },
 391     parent => get_standard_option('pve-snapshot-name', {
 392         optional => 1,
 393         description => "Parent snapshot name. This is used internally, and should not be modified.",
 394     }),
 395     snaptime => {
 396         optional => 1,
 397         description => "Timestamp for snapshots.",
 398         type => 'integer',
 399         minimum => 0,
 400     },
 401     vmstate => {
 402         optional => 1,
 403         type => 'string', format => 'pve-volume-id',
 404         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 405     },
 406     machine => {
 407         description => "Specific the Qemu machine type.",
 408         type => 'string',
 409         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 410         maxLength => 40,
 411         optional => 1,
 412     },
 413     smbios1 => {
 414         description => "Specify SMBIOS type 1 fields.",
 415         type => 'string', format => 'pve-qm-smbios1',
 416         maxLength => 256,
 417         optional => 1,
 418     },
 419     protection => {
 420         optional => 1,
 421         type => 'boolean',
 422         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 423         default => 0,
 424     },
 425     bios => {
 426         optional => 1,
 427         type => 'string',
 428         enum => [ qw(seabios ovmf) ],
 429         description => "Select BIOS implementation.",
 430         default => 'seabios',
 431     },
 432 };
 433
 434 # what about other qemu settings ?
 435 #cpu => 'string',
 436 #machine => 'string',
 437 #fda => 'file',
 438 #fdb => 'file',
 439 #mtdblock => 'file',
 440 #sd => 'file',
 441 #pflash => 'file',
 442 #snapshot => 'bool',
 443 #bootp => 'file',
 444 ##tftp => 'dir',
 445 ##smb => 'dir',
 446 #kernel => 'file',
 447 #append => 'string',
 448 #initrd => 'file',
 449 ##soundhw => 'string',
 450
 451 while (my ($k, $v) = each %$confdesc) {
 452     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 453 }
 454
 455 my $MAX_IDE_DISKS = 4;
 456 my $MAX_SCSI_DISKS = 14;
 457 my $MAX_VIRTIO_DISKS = 16;
 458 my $MAX_SATA_DISKS = 6;
 459 my $MAX_USB_DEVICES = 5;
 460 my $MAX_NETS = 32;
 461 my $MAX_UNUSED_DISKS = 8;
 462 my $MAX_HOSTPCI_DEVICES = 4;
 463 my $MAX_SERIAL_PORTS = 4;
 464 my $MAX_PARALLEL_PORTS = 3;
 465 my $MAX_NUMA = 8;
 466 my $MAX_MEM = 4194304;
 467 my $STATICMEM = 1024;
 468
 469 my $numadesc = {
 470     optional => 1,
 471     type => 'string', format => 'pve-qm-numanode',
 472     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 473     description => "numa topology",
 474 };
 475 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 476
 477 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 478     $confdesc->{"numa$i"} = $numadesc;
 479 }
 480
 481 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 482                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 483                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 484 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 485
 486 my $netdesc = {
 487     optional => 1,
 488     type => 'string', format => 'pve-qm-net',
 489     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 490     description => <<EODESCR,
 491 Specify network devices.
 492
 493 MODEL is one of: $nic_model_list_txt
 494
 495 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 496 automatically generated if not specified.
 497
 498 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 499
 500 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 501
 502 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 503
 504 10.0.2.2   Gateway
 505 10.0.2.3   DNS Server
 506 10.0.2.4   SMB Server
 507
 508 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 509
 510 EODESCR
 511 };
 512 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 513
 514 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 515     $confdesc->{"net$i"} = $netdesc;
 516 }
 517
 518 my $drivename_hash;
 519
 520 my %drivedesc_base = (
 521     volume => { alias => 'file' },
 522     file => {
 523         type => 'pve-volume-id',
 524         default_key => 1,
 525         format_description => 'volume',
 526         description => "The drive's backing volume.",
 527     },
 528     media => {
 529         type => 'string',
 530         format_description => 'cdrom|disk',
 531         enum => [qw(cdrom disk)],
 532         description => "The drive's media type.",
 533         default => 'disk',
 534         optional => 1
 535     },
 536     cyls => {
 537         type => 'integer',
 538         format_description => 'count',
 539         description => "Force the drive's physical geometry to have a specific cylinder count.",
 540         optional => 1
 541     },
 542     heads => {
 543         type => 'integer',
 544         format_description => 'count',
 545         description => "Force the drive's physical geometry to have a specific head count.",
 546         optional => 1
 547     },
 548     secs => {
 549         type => 'integer',
 550         format_description => 'count',
 551         description => "Force the drive's physical geometry to have a specific sector count.",
 552         optional => 1
 553     },
 554     trans => {
 555         type => 'string',
 556         format_description => 'none|lba|auto',
 557         enum => [qw(none lba auto)],
 558         description => "Force disk geometry bios translation mode.",
 559         optional => 1,
 560     },
 561     snapshot => {
 562         type => 'boolean',
 563         format_description => 'on|off',
 564         description => "Whether the drive should be included when making snapshots.",
 565         optional => 1,
 566     },
 567     cache => {
 568         type => 'string',
 569         format_description => 'none|writethrough|writeback|unsafe|directsync',
 570         enum => [qw(none writethrough writeback unsafe directsync)],
 571         description => "The drive's cache mode",
 572         optional => 1,
 573     },
 574     format => {
 575         type => 'string',
 576         format_description => 'drive format',
 577         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 578         description => "The drive's backing file's data format.",
 579         optional => 1,
 580     },
 581     size => {
 582         type => 'string',
 583         format => 'disk-size',
 584         description => "Disk size. This is purely informational and has no effect.",
 585         optional => 1,
 586     },
 587     backup => {
 588         type => 'boolean',
 589         format_description => 'on|off',
 590         description => "Whether the drive should be included when making backups.",
 591         optional => 1,
 592     },
 593     werror => {
 594         type => 'string',
 595         format_description => 'enospc|ignore|report|stop',
 596         enum => [qw(enospc ignore report stop)],
 597         description => 'Write error action.',
 598         optional => 1,
 599     },
 600     aio => {
 601         type => 'string',
 602         format_description => 'native|threads',
 603         enum => [qw(native threads)],
 604         description => 'AIO type to use.',
 605         optional => 1,
 606     },
 607     discard => {
 608         type => 'string',
 609         format_description => 'ignore|on',
 610         enum => [qw(ignore on)],
 611         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 612         optional => 1,
 613     },
 614     detect_zeroes => {
 615         type => 'boolean',
 616         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 617         optional => 1,
 618     },
 619     serial => {
 620         type => 'string',
 621         format => 'urlencoded',
 622         format_description => 'serial',
 623         description => "The drive's reported serial number, url-encoded.",
 624         optional => 1,
 625     }
 626 );
 627
 628 my %rerror_fmt = (
 629     rerror => {
 630         type => 'string',
 631         format_description => 'ignore|report|stop',
 632         enum => [qw(ignore report stop)],
 633         description => 'Read error action.',
 634         optional => 1,
 635     },
 636 );
 637
 638 my %iothread_fmt = ( iothread => {
 639         type => 'boolean',
 640         format_description => 'off|on',
 641         description => "Whether to use iothreads for this drive",
 642         optional => 1,
 643 });
 644
 645 my %model_fmt = (
 646     model => {
 647         type => 'string',
 648         format => 'urlencoded',
 649         format_description => 'model',
 650         description => "The drive's reported model name, url-encoded.",
 651         optional => 1,
 652     },
 653 );
 654
 655 my %queues_fmt = (
 656     queues => {
 657         type => 'integer',
 658         format_description => 'nbqueues',
 659         description => "Number of queues.",
 660         minimum => 2,
 661         optional => 1
 662     }
 663 );
 664
 665 my $add_throttle_desc = sub {
 666     my ($key, $type, $what, $size, $longsize) = @_;
 667     $drivedesc_base{$key} = {
 668         type => $type,
 669         format_description => $size,
 670         description => "Maximum $what speed in $longsize per second.",
 671         optional => 1,
 672     };
 673 };
 674 # throughput: (leaky bucket)
 675 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 676 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 677 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 678 $add_throttle_desc->('mbps',    'float',   'r/w speed',   'mbps', 'megabytes');
 679 $add_throttle_desc->('mbps_rd', 'float',   'read speed',  'mbps', 'megabytes');
 680 $add_throttle_desc->('mbps_wr', 'float',   'write speed', 'mbps', 'megabytes');
 681 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 682 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 683 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 684
 685 # pools: (pool of IO before throttling starts taking effect)
 686 $add_throttle_desc->('mbps_max',    'float',   'unthrottled r/w pool',       'mbps', 'megabytes');
 687 $add_throttle_desc->('mbps_rd_max', 'float',   'unthrottled read pool',      'mbps', 'megabytes');
 688 $add_throttle_desc->('mbps_wr_max', 'float',   'unthrottled write pool',     'mbps', 'megabytes');
 689 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 690 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 691 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 692
 693 my $ide_fmt = {
 694     %drivedesc_base,
 695     %rerror_fmt,
 696     %model_fmt,
 697 };
 698
 699 my $idedesc = {
 700     optional => 1,
 701     type => 'string', format => $ide_fmt,
 702     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 703 };
 704 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 705
 706 my $scsi_fmt = {
 707     %drivedesc_base,
 708     %iothread_fmt,
 709     %queues_fmt,
 710 };
 711 my $scsidesc = {
 712     optional => 1,
 713     type => 'string', format => $scsi_fmt,
 714     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 715 };
 716 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 717
 718 my $sata_fmt = {
 719     %drivedesc_base,
 720     %rerror_fmt,
 721 };
 722 my $satadesc = {
 723     optional => 1,
 724     type => 'string', format => $sata_fmt,
 725     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 726 };
 727 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 728
 729 my $virtio_fmt = {
 730     %drivedesc_base,
 731     %iothread_fmt,
 732     %rerror_fmt,
 733 };
 734 my $virtiodesc = {
 735     optional => 1,
 736     type => 'string', format => $virtio_fmt,
 737     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 738 };
 739 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 740
 741 my $alldrive_fmt = {
 742     %drivedesc_base,
 743     %rerror_fmt,
 744     %iothread_fmt,
 745     %model_fmt,
 746     %queues_fmt,
 747 };
 748
 749 my $usbformat = {
 750     host => {
 751         default_key => 1,
 752         type => 'string', format => 'pve-qm-usb-device',
 753         format_description => 'HOSTUSBDEVICE|spice',
 754         description => 'The Host USB device or port or the value spice',
 755     },
 756     usb3 => {
 757         optional => 1,
 758         type => 'boolean',
 759         format_description => 'yes|no',
 760         description => 'Specifies whether if given host option is a USB3 device or port',
 761     },
 762 };
 763
 764 my $usbdesc = {
 765     optional => 1,
 766     type => 'string', format => $usbformat,
 767     description => <<EODESCR,
 768 Configure an USB device (n is 0 to 4). This can be used to
 769 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 770
 771 'bus-port(.port)*' (decimal numbers) or
 772 'vendor_id:product_id' (hexadeciaml numbers) or
 773 'spice'
 774
 775 You can use the 'lsusb -t' command to list existing usb devices.
 776
 777 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 778
 779 The value 'spice' can be used to add a usb redirection devices for spice.
 780
 781 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 782
 783 EODESCR
 784 };
 785 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 786
 787 my $hostpcidesc = {
 788         optional => 1,
 789         type => 'string', format => 'pve-qm-hostpci',
 790         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 791         description => <<EODESCR,
 792 Map host pci devices. HOSTPCIDEVICE syntax is:
 793
 794 'bus:dev.func' (hexadecimal numbers)
 795
 796 You can us the 'lspci' command to list existing pci devices.
 797
 798 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 799
 800 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 801
 802 Experimental: user reported problems with this option.
 803 EODESCR
 804 };
 805 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 806
 807 my $serialdesc = {
 808         optional => 1,
 809         type => 'string',
 810         pattern => '(/dev/.+|socket)',
 811         description =>  <<EODESCR,
 812 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 813
 814 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 815
 816 Experimental: user reported problems with this option.
 817 EODESCR
 818 };
 819
 820 my $paralleldesc= {
 821         optional => 1,
 822         type => 'string',
 823         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 824         description =>  <<EODESCR,
 825 Map host parallel devices (n is 0 to 2).
 826
 827 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 828
 829 Experimental: user reported problems with this option.
 830 EODESCR
 831 };
 832
 833 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 834     $confdesc->{"parallel$i"} = $paralleldesc;
 835 }
 836
 837 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 838     $confdesc->{"serial$i"} = $serialdesc;
 839 }
 840
 841 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 842     $confdesc->{"hostpci$i"} = $hostpcidesc;
 843 }
 844
 845 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 846     $drivename_hash->{"ide$i"} = 1;
 847     $confdesc->{"ide$i"} = $idedesc;
 848 }
 849
 850 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 851     $drivename_hash->{"sata$i"} = 1;
 852     $confdesc->{"sata$i"} = $satadesc;
 853 }
 854
 855 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 856     $drivename_hash->{"scsi$i"} = 1;
 857     $confdesc->{"scsi$i"} = $scsidesc ;
 858 }
 859
 860 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 861     $drivename_hash->{"virtio$i"} = 1;
 862     $confdesc->{"virtio$i"} = $virtiodesc;
 863 }
 864
 865 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 866     $confdesc->{"usb$i"} = $usbdesc;
 867 }
 868
 869 my $unuseddesc = {
 870     optional => 1,
 871     type => 'string', format => 'pve-volume-id',
 872     description => "Reference to unused volumes.",
 873 };
 874
 875 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 876     $confdesc->{"unused$i"} = $unuseddesc;
 877 }
 878
 879 my $kvm_api_version = 0;
 880
 881 sub kvm_version {
 882
 883     return $kvm_api_version if $kvm_api_version;
 884
 885     my $fh = IO::File->new("</dev/kvm") ||
 886         return 0;
 887
 888     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 889         $kvm_api_version = $v;
 890     }
 891
 892     $fh->close();
 893
 894     return  $kvm_api_version;
 895 }
 896
 897 my $kvm_user_version;
 898
 899 sub kvm_user_version {
 900
 901     return $kvm_user_version if $kvm_user_version;
 902
 903     $kvm_user_version = 'unknown';
 904
 905     my $code = sub {
 906         my $line = shift;
 907         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 908             $kvm_user_version = $2;
 909         }
 910     };
 911
 912     eval { run_command("kvm -version", outfunc => $code); };
 913     warn $@ if $@;
 914
 915     return $kvm_user_version;
 916
 917 }
 918
 919 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 920
 921 sub valid_drive_names {
 922     # order is important - used to autoselect boot disk
 923     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 924             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 925             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 926             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 927 }
 928
 929 sub is_valid_drivename {
 930     my $dev = shift;
 931
 932     return defined($drivename_hash->{$dev});
 933 }
 934
 935 sub option_exists {
 936     my $key = shift;
 937     return defined($confdesc->{$key});
 938 }
 939
 940 sub nic_models {
 941     return $nic_model_list;
 942 }
 943
 944 sub os_list_description {
 945
 946     return {
 947         other => 'Other',
 948         wxp => 'Windows XP',
 949         w2k => 'Windows 2000',
 950         w2k3 =>, 'Windows 2003',
 951         w2k8 => 'Windows 2008',
 952         wvista => 'Windows Vista',
 953         win7 => 'Windows 7',
 954         win8 => 'Windows 8/2012',
 955         l24 => 'Linux 2.4',
 956         l26 => 'Linux 2.6',
 957     };
 958 }
 959
 960 my $cdrom_path;
 961
 962 sub get_cdrom_path {
 963
 964     return  $cdrom_path if $cdrom_path;
 965
 966     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 967     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 968     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 969 }
 970
 971 sub get_iso_path {
 972     my ($storecfg, $vmid, $cdrom) = @_;
 973
 974     if ($cdrom eq 'cdrom') {
 975         return get_cdrom_path();
 976     } elsif ($cdrom eq 'none') {
 977         return '';
 978     } elsif ($cdrom =~ m|^/|) {
 979         return $cdrom;
 980     } else {
 981         return PVE::Storage::path($storecfg, $cdrom);
 982     }
 983 }
 984
 985 # try to convert old style file names to volume IDs
 986 sub filename_to_volume_id {
 987     my ($vmid, $file, $media) = @_;
 988
 989     if (!($file eq 'none' || $file eq 'cdrom' ||
 990           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 991
 992         return undef if $file =~ m|/|;
 993
 994         if ($media && $media eq 'cdrom') {
 995             $file = "local:iso/$file";
 996         } else {
 997             $file = "local:$vmid/$file";
 998         }
 999     }
1000
1001     return $file;
1002 }
1003
1004 sub verify_media_type {
1005     my ($opt, $vtype, $media) = @_;
1006
1007     return if !$media;
1008
1009     my $etype;
1010     if ($media eq 'disk') {
1011         $etype = 'images';
1012     } elsif ($media eq 'cdrom') {
1013         $etype = 'iso';
1014     } else {
1015         die "internal error";
1016     }
1017
1018     return if ($vtype eq $etype);
1019
1020     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1021 }
1022
1023 sub cleanup_drive_path {
1024     my ($opt, $storecfg, $drive) = @_;
1025
1026     # try to convert filesystem paths to volume IDs
1027
1028     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1029         ($drive->{file} !~ m|^/dev/.+|) &&
1030         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1031         ($drive->{file} !~ m/^\d+$/)) {
1032         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1033         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1034         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1035         verify_media_type($opt, $vtype, $drive->{media});
1036         $drive->{file} = $volid;
1037     }
1038
1039     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1040 }
1041
1042 sub parse_hotplug_features {
1043     my ($data) = @_;
1044
1045     my $res = {};
1046
1047     return $res if $data eq '0';
1048
1049     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1050
1051     foreach my $feature (PVE::Tools::split_list($data)) {
1052         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1053             $res->{$1} = 1;
1054         } else {
1055             warn "ignoring unknown hotplug feature '$feature'\n";
1056         }
1057     }
1058     return $res;
1059 }
1060
1061 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1062 sub pve_verify_hotplug_features {
1063     my ($value, $noerr) = @_;
1064
1065     return $value if parse_hotplug_features($value);
1066
1067     return undef if $noerr;
1068
1069     die "unable to parse hotplug option\n";
1070 }
1071
1072 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1073 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1074 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1075 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1076 #        [,iothread=on][,serial=serial][,model=model]
1077
1078 sub parse_drive {
1079     my ($key, $data) = @_;
1080
1081     my ($interface, $index);
1082
1083     if ($key =~ m/^([^\d]+)(\d+)$/) {
1084         $interface = $1;
1085         $index = $2;
1086     } else {
1087         return undef;
1088     }
1089
1090     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1091                                      : $confdesc->{$key}->{format};
1092     if (!$desc) {
1093         warn "invalid drive key: $key\n";
1094         return undef;
1095     }
1096     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1097     return undef if !$res;
1098     $res->{interface} = $interface;
1099     $res->{index} = $index;
1100
1101     my $error = 0;
1102     foreach my $opt (qw(bps bps_rd bps_wr)) {
1103         if (my $bps = defined(delete $res->{$opt})) {
1104             if (defined($res->{"m$opt"})) {
1105                 warn "both $opt and m$opt specified\n";
1106                 ++$error;
1107                 next;
1108             }
1109             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1110         }
1111     }
1112     return undef if $error;
1113
1114     return undef if $res->{mbps_rd} && $res->{mbps};
1115     return undef if $res->{mbps_wr} && $res->{mbps};
1116     return undef if $res->{iops_rd} && $res->{iops};
1117     return undef if $res->{iops_wr} && $res->{iops};
1118
1119     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1120         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1121         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1122         return undef if $res->{interface} eq 'virtio';
1123     }
1124
1125     if (my $size = $res->{size}) {
1126         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1127     }
1128
1129     return $res;
1130 }
1131
1132 sub print_drive {
1133     my ($vmid, $drive) = @_;
1134     my $data = { %$drive };
1135     delete $data->{$_} for qw(index interface);
1136     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1137 }
1138
1139 sub scsi_inquiry {
1140     my($fh, $noerr) = @_;
1141
1142     my $SG_IO = 0x2285;
1143     my $SG_GET_VERSION_NUM = 0x2282;
1144
1145     my $versionbuf = "\x00" x 8;
1146     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1147     if (!$ret) {
1148         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1149         return undef;
1150     }
1151     my $version = unpack("I", $versionbuf);
1152     if ($version < 30000) {
1153         die "scsi generic interface too old\n"  if !$noerr;
1154         return undef;
1155     }
1156
1157     my $buf = "\x00" x 36;
1158     my $sensebuf = "\x00" x 8;
1159     my $cmd = pack("C x3 C x1", 0x12, 36);
1160
1161     # see /usr/include/scsi/sg.h
1162     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1163
1164     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1165                       length($sensebuf), 0, length($buf), $buf,
1166                       $cmd, $sensebuf, 6000);
1167
1168     $ret = ioctl($fh, $SG_IO, $packet);
1169     if (!$ret) {
1170         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1171         return undef;
1172     }
1173
1174     my @res = unpack($sg_io_hdr_t, $packet);
1175     if ($res[17] || $res[18]) {
1176         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1177         return undef;
1178     }
1179
1180     my $res = {};
1181     (my $byte0, my $byte1, $res->{vendor},
1182      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1183
1184     $res->{removable} = $byte1 & 128 ? 1 : 0;
1185     $res->{type} = $byte0 & 31;
1186
1187     return $res;
1188 }
1189
1190 sub path_is_scsi {
1191     my ($path) = @_;
1192
1193     my $fh = IO::File->new("+<$path") || return undef;
1194     my $res = scsi_inquiry($fh, 1);
1195     close($fh);
1196
1197     return $res;
1198 }
1199
1200 sub machine_type_is_q35 {
1201     my ($conf) = @_;
1202
1203     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1204 }
1205
1206 sub print_tabletdevice_full {
1207     my ($conf) = @_;
1208
1209     my $q35 = machine_type_is_q35($conf);
1210
1211     # we use uhci for old VMs because tablet driver was buggy in older qemu
1212     my $usbbus = $q35 ? "ehci" : "uhci";
1213
1214     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1215 }
1216
1217 sub print_drivedevice_full {
1218     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1219
1220     my $device = '';
1221     my $maxdev = 0;
1222
1223     if ($drive->{interface} eq 'virtio') {
1224         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1225         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1226         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1227     } elsif ($drive->{interface} eq 'scsi') {
1228
1229         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1230         my $unit = $drive->{index} % $maxdev;
1231         my $devicetype = 'hd';
1232         my $path = '';
1233         if (drive_is_cdrom($drive)) {
1234             $devicetype = 'cd';
1235         } else {
1236             if ($drive->{file} =~ m|^/|) {
1237                 $path = $drive->{file};
1238                 if (my $info = path_is_scsi($path)) {
1239                     if ($info->{type} == 0) {
1240                         $devicetype = 'block';
1241                     } elsif ($info->{type} == 1) { # tape
1242                         $devicetype = 'generic';
1243                     }
1244                 }
1245             } else {
1246                  $path = PVE::Storage::path($storecfg, $drive->{file});
1247             }
1248
1249             if($path =~ m/^iscsi\:\/\//){
1250                 $devicetype = 'generic';
1251             }
1252         }
1253
1254         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1255             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1256         } else {
1257             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1258         }
1259
1260     } elsif ($drive->{interface} eq 'ide'){
1261         $maxdev = 2;
1262         my $controller = int($drive->{index} / $maxdev);
1263         my $unit = $drive->{index} % $maxdev;
1264         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1265
1266         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1267         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1268             $model = URI::Escape::uri_unescape($model);
1269             $device .= ",model=$model";
1270         }
1271     } elsif ($drive->{interface} eq 'sata'){
1272         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1273         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1274         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1275     } elsif ($drive->{interface} eq 'usb') {
1276         die "implement me";
1277         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1278     } else {
1279         die "unsupported interface type";
1280     }
1281
1282     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1283
1284     return $device;
1285 }
1286
1287 sub get_initiator_name {
1288     my $initiator;
1289
1290     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1291     while (defined(my $line = <$fh>)) {
1292         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1293         $initiator = $1;
1294         last;
1295     }
1296     $fh->close();
1297
1298     return $initiator;
1299 }
1300
1301 sub print_drive_full {
1302     my ($storecfg, $vmid, $drive) = @_;
1303
1304     my $path;
1305     my $volid = $drive->{file};
1306     my $format;
1307
1308     if (drive_is_cdrom($drive)) {
1309         $path = get_iso_path($storecfg, $vmid, $volid);
1310     } else {
1311         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1312         if ($storeid) {
1313             $path = PVE::Storage::path($storecfg, $volid);
1314             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1315             $format = qemu_img_format($scfg, $volname);
1316         } else {
1317             $path = $volid;
1318             $format = "raw";
1319         }
1320    }
1321
1322     my $opts = '';
1323     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1324     foreach my $o (@qemu_drive_options) {
1325         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1326     }
1327     if (my $serial = $drive->{serial}) {
1328         $serial = URI::Escape::uri_unescape($serial);
1329         $opts .= ",serial=$serial";
1330     }
1331
1332     $opts .= ",format=$format" if $format && !$drive->{format};
1333
1334     foreach my $o (qw(bps bps_rd bps_wr)) {
1335         my $v = $drive->{"m$o"};
1336         $opts .= ",$o=" . int($v*1024*1024) if $v;
1337     }
1338
1339     my $cache_direct = 0;
1340
1341     if (my $cache = $drive->{cache}) {
1342         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1343     } elsif (!drive_is_cdrom($drive)) {
1344         $opts .= ",cache=none";
1345         $cache_direct = 1;
1346     }
1347
1348     # aio native works only with O_DIRECT
1349     if (!$drive->{aio}) {
1350         if($cache_direct) {
1351             $opts .= ",aio=native";
1352         } else {
1353             $opts .= ",aio=threads";
1354         }
1355     }
1356
1357     if (!drive_is_cdrom($drive)) {
1358         my $detectzeroes;
1359         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1360             $detectzeroes = 'off';
1361         } elsif ($drive->{discard}) {
1362             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1363         } else {
1364             # This used to be our default with discard not being specified:
1365             $detectzeroes = 'on';
1366         }
1367         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1368     }
1369
1370     my $pathinfo = $path ? "file=$path," : '';
1371
1372     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1373 }
1374
1375 sub print_netdevice_full {
1376     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1377
1378     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1379
1380     my $device = $net->{model};
1381     if ($net->{model} eq 'virtio') {
1382          $device = 'virtio-net-pci';
1383      };
1384
1385     my $pciaddr = print_pci_addr("$netid", $bridges);
1386     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1387     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1388         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1389         my $vectors = $net->{queues} * 2 + 2;
1390         $tmpstr .= ",vectors=$vectors,mq=on";
1391     }
1392     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1393
1394     if ($use_old_bios_files) {
1395         my $romfile;
1396         if ($device eq 'virtio-net-pci') {
1397             $romfile = 'pxe-virtio.rom';
1398         } elsif ($device eq 'e1000') {
1399             $romfile = 'pxe-e1000.rom';
1400         } elsif ($device eq 'ne2k') {
1401             $romfile = 'pxe-ne2k_pci.rom';
1402         } elsif ($device eq 'pcnet') {
1403             $romfile = 'pxe-pcnet.rom';
1404         } elsif ($device eq 'rtl8139') {
1405             $romfile = 'pxe-rtl8139.rom';
1406         }
1407         $tmpstr .= ",romfile=$romfile" if $romfile;
1408     }
1409
1410     return $tmpstr;
1411 }
1412
1413 sub print_netdev_full {
1414     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1415
1416     my $i = '';
1417     if ($netid =~ m/^net(\d+)$/) {
1418         $i = int($1);
1419     }
1420
1421     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1422
1423     my $ifname = "tap${vmid}i$i";
1424
1425     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1426     die "interface name '$ifname' is too long (max 15 character)\n"
1427         if length($ifname) >= 16;
1428
1429     my $vhostparam = '';
1430     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1431
1432     my $vmname = $conf->{name} || "vm$vmid";
1433
1434     my $netdev = "";
1435     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1436
1437     if ($net->{bridge}) {
1438         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1439     } else {
1440         $netdev = "type=user,id=$netid,hostname=$vmname";
1441     }
1442
1443     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1444
1445     return $netdev;
1446 }
1447
1448 sub drive_is_cdrom {
1449     my ($drive) = @_;
1450
1451     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1452
1453 }
1454
1455 sub parse_numa {
1456     my ($data) = @_;
1457
1458     my $res = {};
1459
1460     foreach my $kvp (split(/,/, $data)) {
1461
1462         if ($kvp =~ m/^memory=(\S+)$/) {
1463             $res->{memory} = $1;
1464         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1465             $res->{policy} = $1;
1466         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1467             $res->{cpus}->{start} = $1;
1468             $res->{cpus}->{end} = $3;
1469         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1470             $res->{hostnodes}->{start} = $1;
1471             $res->{hostnodes}->{end} = $3;
1472         } else {
1473             return undef;
1474         }
1475     }
1476
1477     return $res;
1478 }
1479
1480 sub parse_hostpci {
1481     my ($value) = @_;
1482
1483     return undef if !$value;
1484
1485
1486     my @list = split(/,/, $value);
1487     my $found;
1488
1489     my $res = {};
1490     foreach my $kv (@list) {
1491
1492         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1493             $found = 1;
1494             if(defined($4)){
1495                 push @{$res->{pciid}}, { id => $2 , function => $4};
1496
1497             }else{
1498                 my $pcidevices = lspci($2);
1499                 $res->{pciid} = $pcidevices->{$2};
1500             }
1501         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1502             $res->{rombar} = $1;
1503         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1504             $res->{'x-vga'} = $1;
1505         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1506             $res->{pcie} = 1 if $1 == 1;
1507         } else {
1508             warn "unknown hostpci setting '$kv'\n";
1509         }
1510     }
1511
1512     return undef if !$found;
1513
1514     return $res;
1515 }
1516
1517 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1518 sub parse_net {
1519     my ($data) = @_;
1520
1521     my $res = {};
1522
1523     foreach my $kvp (split(/,/, $data)) {
1524
1525         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1526             my $model = lc($1);
1527             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1528             $res->{model} = $model;
1529             $res->{macaddr} = $mac;
1530         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1531             $res->{bridge} = $1;
1532         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1533             $res->{queues} = $1;
1534         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1535             $res->{rate} = $1;
1536         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1537             $res->{tag} = $1;
1538         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1539             $res->{trunks} = $1;
1540         } elsif ($kvp =~ m/^firewall=([01])$/) {
1541             $res->{firewall} = $1;
1542         } elsif ($kvp =~ m/^link_down=([01])$/) {
1543             $res->{link_down} = $1;
1544         } else {
1545             return undef;
1546         }
1547
1548     }
1549
1550     return undef if !$res->{model};
1551
1552     return $res;
1553 }
1554
1555 sub print_net {
1556     my $net = shift;
1557
1558     my $res = "$net->{model}";
1559     $res .= "=$net->{macaddr}" if $net->{macaddr};
1560     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1561     $res .= ",rate=$net->{rate}" if $net->{rate};
1562     $res .= ",tag=$net->{tag}" if $net->{tag};
1563     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1564     $res .= ",firewall=1" if $net->{firewall};
1565     $res .= ",link_down=1" if $net->{link_down};
1566     $res .= ",queues=$net->{queues}" if $net->{queues};
1567
1568     return $res;
1569 }
1570
1571 sub add_random_macs {
1572     my ($settings) = @_;
1573
1574     foreach my $opt (keys %$settings) {
1575         next if $opt !~ m/^net(\d+)$/;
1576         my $net = parse_net($settings->{$opt});
1577         next if !$net;
1578         $settings->{$opt} = print_net($net);
1579     }
1580 }
1581
1582 sub vm_is_volid_owner {
1583     my ($storecfg, $vmid, $volid) = @_;
1584
1585     if ($volid !~  m|^/|) {
1586         my ($path, $owner);
1587         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1588         if ($owner && ($owner == $vmid)) {
1589             return 1;
1590         }
1591     }
1592
1593     return undef;
1594 }
1595
1596 sub split_flagged_list {
1597     my $text = shift || '';
1598     $text =~ s/[,;]/ /g;
1599     $text =~ s/^\s+//;
1600     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1601 }
1602
1603 sub join_flagged_list {
1604     my ($how, $lst) = @_;
1605     join $how, map { $lst->{$_} . $_ } keys %$lst;
1606 }
1607
1608 sub vmconfig_delete_pending_option {
1609     my ($conf, $key, $force) = @_;
1610
1611     delete $conf->{pending}->{$key};
1612     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1613     $pending_delete_hash->{$key} = $force ? '!' : '';
1614     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1615 }
1616
1617 sub vmconfig_undelete_pending_option {
1618     my ($conf, $key) = @_;
1619
1620     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1621     delete $pending_delete_hash->{$key};
1622
1623     if (%$pending_delete_hash) {
1624         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1625     } else {
1626         delete $conf->{pending}->{delete};
1627     }
1628 }
1629
1630 sub vmconfig_register_unused_drive {
1631     my ($storecfg, $vmid, $conf, $drive) = @_;
1632
1633     if (!drive_is_cdrom($drive)) {
1634         my $volid = $drive->{file};
1635         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1636             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1637         }
1638     }
1639 }
1640
1641 sub vmconfig_cleanup_pending {
1642     my ($conf) = @_;
1643
1644     # remove pending changes when nothing changed
1645     my $changes;
1646     foreach my $opt (keys %{$conf->{pending}}) {
1647         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1648             $changes = 1;
1649             delete $conf->{pending}->{$opt};
1650         }
1651     }
1652
1653     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1654     my $pending_delete_hash = {};
1655     while (my ($opt, $force) = each %$current_delete_hash) {
1656         if (defined($conf->{$opt})) {
1657             $pending_delete_hash->{$opt} = $force;
1658         } else {
1659             $changes = 1;
1660         }
1661     }
1662
1663     if (%$pending_delete_hash) {
1664         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1665     } else {
1666         delete $conf->{pending}->{delete};
1667     }
1668
1669     return $changes;
1670 }
1671
1672 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1673 my $smbios1_desc = {
1674     uuid => {
1675         type => 'string',
1676         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1677         format_description => 'UUID',
1678         optional => 1,
1679     },
1680     version => {
1681         type => 'string',
1682         pattern => '\S+',
1683         format_description => 'str',
1684         optional => 1,
1685     },
1686     serial => {
1687         type => 'string',
1688         pattern => '\S+',
1689         format_description => 'str',
1690         optional => 1,
1691     },
1692     manufacturer => {
1693         type => 'string',
1694         pattern => '\S+',
1695         format_description => 'name',
1696         optional => 1,
1697     },
1698     product => {
1699         type => 'string',
1700         pattern => '\S+',
1701         format_description => 'name',
1702         optional => 1,
1703     },
1704     sku => {
1705         type => 'string',
1706         pattern => '\S+',
1707         format_description => 'str',
1708         optional => 1,
1709     },
1710     family => {
1711         type => 'string',
1712         pattern => '\S+',
1713         format_description => 'str',
1714         optional => 1,
1715     },
1716 };
1717
1718 sub parse_smbios1 {
1719     my ($data) = @_;
1720
1721     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1722     warn $@ if $@;
1723     return $res;
1724 }
1725
1726 sub print_smbios1 {
1727     my ($smbios1) = @_;
1728     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1729 }
1730
1731 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1732
1733 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1734 sub verify_bootdisk {
1735     my ($value, $noerr) = @_;
1736
1737     return $value if is_valid_drivename($value);
1738
1739     return undef if $noerr;
1740
1741     die "invalid boot disk '$value'\n";
1742 }
1743
1744 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1745 sub verify_numa {
1746     my ($value, $noerr) = @_;
1747
1748     return $value if parse_numa($value);
1749
1750     return undef if $noerr;
1751
1752     die "unable to parse numa options\n";
1753 }
1754
1755 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1756 sub verify_net {
1757     my ($value, $noerr) = @_;
1758
1759     return $value if parse_net($value);
1760
1761     return undef if $noerr;
1762
1763     die "unable to parse network options\n";
1764 }
1765
1766 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1767 sub verify_hostpci {
1768     my ($value, $noerr) = @_;
1769
1770     return $value if parse_hostpci($value);
1771
1772     return undef if $noerr;
1773
1774     die "unable to parse pci id\n";
1775 }
1776
1777 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1778 sub verify_watchdog {
1779     my ($value, $noerr) = @_;
1780
1781     return $value if parse_watchdog($value);
1782
1783     return undef if $noerr;
1784
1785     die "unable to parse watchdog options\n";
1786 }
1787
1788 sub parse_watchdog {
1789     my ($value) = @_;
1790
1791     return undef if !$value;
1792
1793     my $res = {};
1794
1795     foreach my $p (split(/,/, $value)) {
1796         next if $p =~ m/^\s*$/;
1797
1798         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1799             $res->{model} = $2;
1800         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1801             $res->{action} = $2;
1802         } else {
1803             return undef;
1804         }
1805     }
1806
1807     return $res;
1808 }
1809
1810 sub parse_usb_device {
1811     my ($value) = @_;
1812
1813     return undef if !$value;
1814
1815     my $res = {};
1816     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1817         $res->{vendorid} = $2;
1818         $res->{productid} = $4;
1819     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1820         $res->{hostbus} = $1;
1821         $res->{hostport} = $2;
1822     } elsif ($value =~ m/^spice$/i) {
1823         $res->{spice} = 1;
1824     } else {
1825         return undef;
1826     }
1827
1828     return $res;
1829 }
1830
1831 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1832 sub verify_usb_device {
1833     my ($value, $noerr) = @_;
1834
1835     return $value if parse_usb_device($value);
1836
1837     return undef if $noerr;
1838
1839     die "unable to parse usb device\n";
1840 }
1841
1842 # add JSON properties for create and set function
1843 sub json_config_properties {
1844     my $prop = shift;
1845
1846     foreach my $opt (keys %$confdesc) {
1847         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1848         $prop->{$opt} = $confdesc->{$opt};
1849     }
1850
1851     return $prop;
1852 }
1853
1854 sub check_type {
1855     my ($key, $value) = @_;
1856
1857     die "unknown setting '$key'\n" if !$confdesc->{$key};
1858
1859     my $type = $confdesc->{$key}->{type};
1860
1861     if (!defined($value)) {
1862         die "got undefined value\n";
1863     }
1864
1865     if ($value =~ m/[\n\r]/) {
1866         die "property contains a line feed\n";
1867     }
1868
1869     if ($type eq 'boolean') {
1870         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1871         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1872         die "type check ('boolean') failed - got '$value'\n";
1873     } elsif ($type eq 'integer') {
1874         return int($1) if $value =~ m/^(\d+)$/;
1875         die "type check ('integer') failed - got '$value'\n";
1876     } elsif ($type eq 'number') {
1877         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1878         die "type check ('number') failed - got '$value'\n";
1879     } elsif ($type eq 'string') {
1880         if (my $fmt = $confdesc->{$key}->{format}) {
1881             if ($fmt eq 'pve-qm-drive') {
1882                 # special case - we need to pass $key to parse_drive()
1883                 my $drive = parse_drive($key, $value);
1884                 return $value if $drive;
1885                 die "unable to parse drive options\n";
1886             }
1887             PVE::JSONSchema::check_format($fmt, $value);
1888             return $value;
1889         }
1890         $value =~ s/^\"(.*)\"$/$1/;
1891         return $value;
1892     } else {
1893         die "internal error"
1894     }
1895 }
1896
1897 sub check_iommu_support{
1898     #fixme : need to check IOMMU support
1899     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1900
1901     my $iommu=1;
1902     return $iommu;
1903
1904 }
1905
1906 sub touch_config {
1907     my ($vmid) = @_;
1908
1909     my $conf = PVE::QemuConfig->config_file($vmid);
1910     utime undef, undef, $conf;
1911 }
1912
1913 sub destroy_vm {
1914     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1915
1916     my $conffile = PVE::QemuConfig->config_file($vmid);
1917
1918     my $conf = PVE::QemuConfig->load_config($vmid);
1919
1920     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1921
1922     # only remove disks owned by this VM
1923     foreach_drive($conf, sub {
1924         my ($ds, $drive) = @_;
1925
1926         return if drive_is_cdrom($drive);
1927
1928         my $volid = $drive->{file};
1929
1930         return if !$volid || $volid =~ m|^/|;
1931
1932         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1933         return if !$path || !$owner || ($owner != $vmid);
1934
1935         PVE::Storage::vdisk_free($storecfg, $volid);
1936     });
1937
1938     if ($keep_empty_config) {
1939         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1940     } else {
1941         unlink $conffile;
1942     }
1943
1944     # also remove unused disk
1945     eval {
1946         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1947
1948         eval {
1949             PVE::Storage::foreach_volid($dl, sub {
1950                 my ($volid, $sid, $volname, $d) = @_;
1951                 PVE::Storage::vdisk_free($storecfg, $volid);
1952             });
1953         };
1954         warn $@ if $@;
1955
1956     };
1957     warn $@ if $@;
1958 }
1959
1960 sub parse_vm_config {
1961     my ($filename, $raw) = @_;
1962
1963     return undef if !defined($raw);
1964
1965     my $res = {
1966         digest => Digest::SHA::sha1_hex($raw),
1967         snapshots => {},
1968         pending => {},
1969     };
1970
1971     $filename =~ m|/qemu-server/(\d+)\.conf$|
1972         || die "got strange filename '$filename'";
1973
1974     my $vmid = $1;
1975
1976     my $conf = $res;
1977     my $descr;
1978     my $section = '';
1979
1980     my @lines = split(/\n/, $raw);
1981     foreach my $line (@lines) {
1982         next if $line =~ m/^\s*$/;
1983
1984         if ($line =~ m/^\[PENDING\]\s*$/i) {
1985             $section = 'pending';
1986             if (defined($descr)) {
1987                 $descr =~ s/\s+$//;
1988                 $conf->{description} = $descr;
1989             }
1990             $descr = undef;
1991             $conf = $res->{$section} = {};
1992             next;
1993
1994         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1995             $section = $1;
1996             if (defined($descr)) {
1997                 $descr =~ s/\s+$//;
1998                 $conf->{description} = $descr;
1999             }
2000             $descr = undef;
2001             $conf = $res->{snapshots}->{$section} = {};
2002             next;
2003         }
2004
2005         if ($line =~ m/^\#(.*)\s*$/) {
2006             $descr = '' if !defined($descr);
2007             $descr .= PVE::Tools::decode_text($1) . "\n";
2008             next;
2009         }
2010
2011         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2012             $descr = '' if !defined($descr);
2013             $descr .= PVE::Tools::decode_text($2);
2014         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2015             $conf->{snapstate} = $1;
2016         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2017             my $key = $1;
2018             my $value = $2;
2019             $conf->{$key} = $value;
2020         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2021             my $value = $1;
2022             if ($section eq 'pending') {
2023                 $conf->{delete} = $value; # we parse this later
2024             } else {
2025                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2026             }
2027         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2028             my $key = $1;
2029             my $value = $2;
2030             eval { $value = check_type($key, $value); };
2031             if ($@) {
2032                 warn "vm $vmid - unable to parse value of '$key' - $@";
2033             } else {
2034                 my $fmt = $confdesc->{$key}->{format};
2035                 if ($fmt && $fmt eq 'pve-qm-drive') {
2036                     my $v = parse_drive($key, $value);
2037                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2038                         $v->{file} = $volid;
2039                         $value = print_drive($vmid, $v);
2040                     } else {
2041                         warn "vm $vmid - unable to parse value of '$key'\n";
2042                         next;
2043                     }
2044                 }
2045
2046                 if ($key eq 'cdrom') {
2047                     $conf->{ide2} = $value;
2048                 } else {
2049                     $conf->{$key} = $value;
2050                 }
2051             }
2052         }
2053     }
2054
2055     if (defined($descr)) {
2056         $descr =~ s/\s+$//;
2057         $conf->{description} = $descr;
2058     }
2059     delete $res->{snapstate}; # just to be sure
2060
2061     return $res;
2062 }
2063
2064 sub write_vm_config {
2065     my ($filename, $conf) = @_;
2066
2067     delete $conf->{snapstate}; # just to be sure
2068
2069     if ($conf->{cdrom}) {
2070         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2071         $conf->{ide2} = $conf->{cdrom};
2072         delete $conf->{cdrom};
2073     }
2074
2075     # we do not use 'smp' any longer
2076     if ($conf->{sockets}) {
2077         delete $conf->{smp};
2078     } elsif ($conf->{smp}) {
2079         $conf->{sockets} = $conf->{smp};
2080         delete $conf->{cores};
2081         delete $conf->{smp};
2082     }
2083
2084     my $used_volids = {};
2085
2086     my $cleanup_config = sub {
2087         my ($cref, $pending, $snapname) = @_;
2088
2089         foreach my $key (keys %$cref) {
2090             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2091                 $key eq 'snapstate' || $key eq 'pending';
2092             my $value = $cref->{$key};
2093             if ($key eq 'delete') {
2094                 die "propertry 'delete' is only allowed in [PENDING]\n"
2095                     if !$pending;
2096                 # fixme: check syntax?
2097                 next;
2098             }
2099             eval { $value = check_type($key, $value); };
2100             die "unable to parse value of '$key' - $@" if $@;
2101
2102             $cref->{$key} = $value;
2103
2104             if (!$snapname && is_valid_drivename($key)) {
2105                 my $drive = parse_drive($key, $value);
2106                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2107             }
2108         }
2109     };
2110
2111     &$cleanup_config($conf);
2112
2113     &$cleanup_config($conf->{pending}, 1);
2114
2115     foreach my $snapname (keys %{$conf->{snapshots}}) {
2116         die "internal error" if $snapname eq 'pending';
2117         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2118     }
2119
2120     # remove 'unusedX' settings if we re-add a volume
2121     foreach my $key (keys %$conf) {
2122         my $value = $conf->{$key};
2123         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2124             delete $conf->{$key};
2125         }
2126     }
2127
2128     my $generate_raw_config = sub {
2129         my ($conf, $pending) = @_;
2130
2131         my $raw = '';
2132
2133         # add description as comment to top of file
2134         if (defined(my $descr = $conf->{description})) {
2135             if ($descr) {
2136                 foreach my $cl (split(/\n/, $descr)) {
2137                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2138                 }
2139             } else {
2140                 $raw .= "#\n" if $pending;
2141             }
2142         }
2143
2144         foreach my $key (sort keys %$conf) {
2145             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2146             $raw .= "$key: $conf->{$key}\n";
2147         }
2148         return $raw;
2149     };
2150
2151     my $raw = &$generate_raw_config($conf);
2152
2153     if (scalar(keys %{$conf->{pending}})){
2154         $raw .= "\n[PENDING]\n";
2155         $raw .= &$generate_raw_config($conf->{pending}, 1);
2156     }
2157
2158     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2159         $raw .= "\n[$snapname]\n";
2160         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2161     }
2162
2163     return $raw;
2164 }
2165
2166 sub load_defaults {
2167
2168     my $res = {};
2169
2170     # we use static defaults from our JSON schema configuration
2171     foreach my $key (keys %$confdesc) {
2172         if (defined(my $default = $confdesc->{$key}->{default})) {
2173             $res->{$key} = $default;
2174         }
2175     }
2176
2177     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2178     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2179
2180     return $res;
2181 }
2182
2183 sub config_list {
2184     my $vmlist = PVE::Cluster::get_vmlist();
2185     my $res = {};
2186     return $res if !$vmlist || !$vmlist->{ids};
2187     my $ids = $vmlist->{ids};
2188
2189     foreach my $vmid (keys %$ids) {
2190         my $d = $ids->{$vmid};
2191         next if !$d->{node} || $d->{node} ne $nodename;
2192         next if !$d->{type} || $d->{type} ne 'qemu';
2193         $res->{$vmid}->{exists} = 1;
2194     }
2195     return $res;
2196 }
2197
2198 # test if VM uses local resources (to prevent migration)
2199 sub check_local_resources {
2200     my ($conf, $noerr) = @_;
2201
2202     my $loc_res = 0;
2203
2204     $loc_res = 1 if $conf->{hostusb}; # old syntax
2205     $loc_res = 1 if $conf->{hostpci}; # old syntax
2206
2207     foreach my $k (keys %$conf) {
2208         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2209         # sockets are safe: they will recreated be on the target side post-migrate
2210         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2211         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2212     }
2213
2214     die "VM uses local resources\n" if $loc_res && !$noerr;
2215
2216     return $loc_res;
2217 }
2218
2219 # check if used storages are available on all nodes (use by migrate)
2220 sub check_storage_availability {
2221     my ($storecfg, $conf, $node) = @_;
2222
2223     foreach_drive($conf, sub {
2224         my ($ds, $drive) = @_;
2225
2226         my $volid = $drive->{file};
2227         return if !$volid;
2228
2229         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2230         return if !$sid;
2231
2232         # check if storage is available on both nodes
2233         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2234         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2235    });
2236 }
2237
2238 # list nodes where all VM images are available (used by has_feature API)
2239 sub shared_nodes {
2240     my ($conf, $storecfg) = @_;
2241
2242     my $nodelist = PVE::Cluster::get_nodelist();
2243     my $nodehash = { map { $_ => 1 } @$nodelist };
2244     my $nodename = PVE::INotify::nodename();
2245
2246     foreach_drive($conf, sub {
2247         my ($ds, $drive) = @_;
2248
2249         my $volid = $drive->{file};
2250         return if !$volid;
2251
2252         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2253         if ($storeid) {
2254             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2255             if ($scfg->{disable}) {
2256                 $nodehash = {};
2257             } elsif (my $avail = $scfg->{nodes}) {
2258                 foreach my $node (keys %$nodehash) {
2259                     delete $nodehash->{$node} if !$avail->{$node};
2260                 }
2261             } elsif (!$scfg->{shared}) {
2262                 foreach my $node (keys %$nodehash) {
2263                     delete $nodehash->{$node} if $node ne $nodename
2264                 }
2265             }
2266         }
2267     });
2268
2269     return $nodehash
2270 }
2271
2272 sub check_cmdline {
2273     my ($pidfile, $pid) = @_;
2274
2275     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2276     if (defined($fh)) {
2277         my $line = <$fh>;
2278         $fh->close;
2279         return undef if !$line;
2280         my @param = split(/\0/, $line);
2281
2282         my $cmd = $param[0];
2283         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2284
2285         for (my $i = 0; $i < scalar (@param); $i++) {
2286             my $p = $param[$i];
2287             next if !$p;
2288             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2289                 my $p = $param[$i+1];
2290                 return 1 if $p && ($p eq $pidfile);
2291                 return undef;
2292             }
2293         }
2294     }
2295     return undef;
2296 }
2297
2298 sub check_running {
2299     my ($vmid, $nocheck, $node) = @_;
2300
2301     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2302
2303     die "unable to find configuration file for VM $vmid - no such machine\n"
2304         if !$nocheck && ! -f $filename;
2305
2306     my $pidfile = pidfile_name($vmid);
2307
2308     if (my $fd = IO::File->new("<$pidfile")) {
2309         my $st = stat($fd);
2310         my $line = <$fd>;
2311         close($fd);
2312
2313         my $mtime = $st->mtime;
2314         if ($mtime > time()) {
2315             warn "file '$filename' modified in future\n";
2316         }
2317
2318         if ($line =~ m/^(\d+)$/) {
2319             my $pid = $1;
2320             if (check_cmdline($pidfile, $pid)) {
2321                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2322                     return $pid;
2323                 }
2324             }
2325         }
2326     }
2327
2328     return undef;
2329 }
2330
2331 sub vzlist {
2332
2333     my $vzlist = config_list();
2334
2335     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2336
2337     while (defined(my $de = $fd->read)) {
2338         next if $de !~ m/^(\d+)\.pid$/;
2339         my $vmid = $1;
2340         next if !defined($vzlist->{$vmid});
2341         if (my $pid = check_running($vmid)) {
2342             $vzlist->{$vmid}->{pid} = $pid;
2343         }
2344     }
2345
2346     return $vzlist;
2347 }
2348
2349 sub disksize {
2350     my ($storecfg, $conf) = @_;
2351
2352     my $bootdisk = $conf->{bootdisk};
2353     return undef if !$bootdisk;
2354     return undef if !is_valid_drivename($bootdisk);
2355
2356     return undef if !$conf->{$bootdisk};
2357
2358     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2359     return undef if !defined($drive);
2360
2361     return undef if drive_is_cdrom($drive);
2362
2363     my $volid = $drive->{file};
2364     return undef if !$volid;
2365
2366     return $drive->{size};
2367 }
2368
2369 my $last_proc_pid_stat;
2370
2371 # get VM status information
2372 # This must be fast and should not block ($full == false)
2373 # We only query KVM using QMP if $full == true (this can be slow)
2374 sub vmstatus {
2375     my ($opt_vmid, $full) = @_;
2376
2377     my $res = {};
2378
2379     my $storecfg = PVE::Storage::config();
2380
2381     my $list = vzlist();
2382     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2383
2384     my $cpucount = $cpuinfo->{cpus} || 1;
2385
2386     foreach my $vmid (keys %$list) {
2387         next if $opt_vmid && ($vmid ne $opt_vmid);
2388
2389         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2390         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2391
2392         my $d = {};
2393         $d->{pid} = $list->{$vmid}->{pid};
2394
2395         # fixme: better status?
2396         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2397
2398         my $size = disksize($storecfg, $conf);
2399         if (defined($size)) {
2400             $d->{disk} = 0; # no info available
2401             $d->{maxdisk} = $size;
2402         } else {
2403             $d->{disk} = 0;
2404             $d->{maxdisk} = 0;
2405         }
2406
2407         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2408         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2409         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2410
2411         $d->{name} = $conf->{name} || "VM $vmid";
2412         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2413
2414         if ($conf->{balloon}) {
2415             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2416             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2417         }
2418
2419         $d->{uptime} = 0;
2420         $d->{cpu} = 0;
2421         $d->{mem} = 0;
2422
2423         $d->{netout} = 0;
2424         $d->{netin} = 0;
2425
2426         $d->{diskread} = 0;
2427         $d->{diskwrite} = 0;
2428
2429         $d->{template} = PVE::QemuConfig->is_template($conf);
2430
2431         $res->{$vmid} = $d;
2432     }
2433
2434     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2435     foreach my $dev (keys %$netdev) {
2436         next if $dev !~ m/^tap([1-9]\d*)i/;
2437         my $vmid = $1;
2438         my $d = $res->{$vmid};
2439         next if !$d;
2440
2441         $d->{netout} += $netdev->{$dev}->{receive};
2442         $d->{netin} += $netdev->{$dev}->{transmit};
2443
2444         if ($full) {
2445             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2446             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2447         }
2448
2449     }
2450
2451     my $ctime = gettimeofday;
2452
2453     foreach my $vmid (keys %$list) {
2454
2455         my $d = $res->{$vmid};
2456         my $pid = $d->{pid};
2457         next if !$pid;
2458
2459         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2460         next if !$pstat; # not running
2461
2462         my $used = $pstat->{utime} + $pstat->{stime};
2463
2464         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2465
2466         if ($pstat->{vsize}) {
2467             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2468         }
2469
2470         my $old = $last_proc_pid_stat->{$pid};
2471         if (!$old) {
2472             $last_proc_pid_stat->{$pid} = {
2473                 time => $ctime,
2474                 used => $used,
2475                 cpu => 0,
2476             };
2477             next;
2478         }
2479
2480         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2481
2482         if ($dtime > 1000) {
2483             my $dutime = $used -  $old->{used};
2484
2485             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2486             $last_proc_pid_stat->{$pid} = {
2487                 time => $ctime,
2488                 used => $used,
2489                 cpu => $d->{cpu},
2490             };
2491         } else {
2492             $d->{cpu} = $old->{cpu};
2493         }
2494     }
2495
2496     return $res if !$full;
2497
2498     my $qmpclient = PVE::QMPClient->new();
2499
2500     my $ballooncb = sub {
2501         my ($vmid, $resp) = @_;
2502
2503         my $info = $resp->{'return'};
2504         return if !$info->{max_mem};
2505
2506         my $d = $res->{$vmid};
2507
2508         # use memory assigned to VM
2509         $d->{maxmem} = $info->{max_mem};
2510         $d->{balloon} = $info->{actual};
2511
2512         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2513             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2514             $d->{freemem} = $info->{free_mem};
2515         }
2516
2517         $d->{ballooninfo} = $info;
2518     };
2519
2520     my $blockstatscb = sub {
2521         my ($vmid, $resp) = @_;
2522         my $data = $resp->{'return'} || [];
2523         my $totalrdbytes = 0;
2524         my $totalwrbytes = 0;
2525
2526         for my $blockstat (@$data) {
2527             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2528             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2529
2530             $blockstat->{device} =~ s/drive-//;
2531             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2532         }
2533         $res->{$vmid}->{diskread} = $totalrdbytes;
2534         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2535     };
2536
2537     my $statuscb = sub {
2538         my ($vmid, $resp) = @_;
2539
2540         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2541         # this fails if ballon driver is not loaded, so this must be
2542         # the last commnand (following command are aborted if this fails).
2543         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2544
2545         my $status = 'unknown';
2546         if (!defined($status = $resp->{'return'}->{status})) {
2547             warn "unable to get VM status\n";
2548             return;
2549         }
2550
2551         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2552     };
2553
2554     foreach my $vmid (keys %$list) {
2555         next if $opt_vmid && ($vmid ne $opt_vmid);
2556         next if !$res->{$vmid}->{pid}; # not running
2557         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2558     }
2559
2560     $qmpclient->queue_execute(undef, 1);
2561
2562     foreach my $vmid (keys %$list) {
2563         next if $opt_vmid && ($vmid ne $opt_vmid);
2564         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2565     }
2566
2567     return $res;
2568 }
2569
2570 sub foreach_dimm {
2571     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2572
2573     my $dimm_id = 0;
2574     my $current_size = 1024;
2575     my $dimm_size = 512;
2576     return if $current_size == $memory;
2577
2578     for (my $j = 0; $j < 8; $j++) {
2579         for (my $i = 0; $i < 32; $i++) {
2580             my $name = "dimm${dimm_id}";
2581             $dimm_id++;
2582             my $numanode = $i % $sockets;
2583             $current_size += $dimm_size;
2584             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2585             return  $current_size if $current_size >= $memory;
2586         }
2587         $dimm_size *= 2;
2588     }
2589 }
2590
2591 sub foreach_reverse_dimm {
2592     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2593
2594     my $dimm_id = 253;
2595     my $current_size = 4177920;
2596     my $dimm_size = 65536;
2597     return if $current_size == $memory;
2598
2599     for (my $j = 0; $j < 8; $j++) {
2600         for (my $i = 0; $i < 32; $i++) {
2601             my $name = "dimm${dimm_id}";
2602             $dimm_id--;
2603             my $numanode = $i % $sockets;
2604             $current_size -= $dimm_size;
2605             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2606             return  $current_size if $current_size <= $memory;
2607         }
2608         $dimm_size /= 2;
2609     }
2610 }
2611
2612 sub foreach_drive {
2613     my ($conf, $func) = @_;
2614
2615     foreach my $ds (valid_drive_names()) {
2616         next if !defined($conf->{$ds});
2617
2618         my $drive = parse_drive($ds, $conf->{$ds});
2619         next if !$drive;
2620
2621         &$func($ds, $drive);
2622     }
2623 }
2624
2625 sub foreach_volid {
2626     my ($conf, $func) = @_;
2627
2628     my $volhash = {};
2629
2630     my $test_volid = sub {
2631         my ($volid, $is_cdrom) = @_;
2632
2633         return if !$volid;
2634
2635         $volhash->{$volid} = $is_cdrom || 0;
2636     };
2637
2638     foreach_drive($conf, sub {
2639         my ($ds, $drive) = @_;
2640         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2641     });
2642
2643     foreach my $snapname (keys %{$conf->{snapshots}}) {
2644         my $snap = $conf->{snapshots}->{$snapname};
2645         &$test_volid($snap->{vmstate}, 0);
2646         foreach_drive($snap, sub {
2647             my ($ds, $drive) = @_;
2648             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2649         });
2650     }
2651
2652     foreach my $volid (keys %$volhash) {
2653         &$func($volid, $volhash->{$volid});
2654     }
2655 }
2656
2657 sub vga_conf_has_spice {
2658     my ($vga) = @_;
2659
2660     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2661
2662     return $1 || 1;
2663 }
2664
2665 sub config_to_command {
2666     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2667
2668     my $cmd = [];
2669     my $globalFlags = [];
2670     my $machineFlags = [];
2671     my $rtcFlags = [];
2672     my $cpuFlags = [];
2673     my $devices = [];
2674     my $pciaddr = '';
2675     my $bridges = {};
2676     my $kvmver = kvm_user_version();
2677     my $vernum = 0; # unknown
2678     my $ostype = $conf->{ostype};
2679     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2680         $vernum = $1*1000000+$2*1000;
2681     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2682         $vernum = $1*1000000+$2*1000+$3;
2683     }
2684
2685     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2686
2687     my $have_ovz = -f '/proc/vz/vestat';
2688
2689     my $q35 = machine_type_is_q35($conf);
2690     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2691     my $machine_type = $forcemachine || $conf->{machine};
2692     my $use_old_bios_files = undef;
2693     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2694
2695     my $cpuunits = defined($conf->{cpuunits}) ?
2696             $conf->{cpuunits} : $defaults->{cpuunits};
2697
2698     push @$cmd, '/usr/bin/systemd-run';
2699     push @$cmd, '--scope';
2700     push @$cmd, '--slice', "qemu";
2701     push @$cmd, '--unit', $vmid;
2702     # set KillMode=none, so that systemd don't kill those scopes
2703     # at shutdown (pve-manager service should stop the VMs instead)
2704     push @$cmd, '-p', "KillMode=none";
2705     push @$cmd, '-p', "CPUShares=$cpuunits";
2706     if ($conf->{cpulimit}) {
2707         my $cpulimit = int($conf->{cpulimit} * 100);
2708         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2709     }
2710
2711     push @$cmd, '/usr/bin/kvm';
2712
2713     push @$cmd, '-id', $vmid;
2714
2715     my $use_virtio = 0;
2716
2717     my $qmpsocket = qmp_socket($vmid);
2718     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2719     push @$cmd, '-mon', "chardev=qmp,mode=control";
2720
2721
2722     push @$cmd, '-pidfile' , pidfile_name($vmid);
2723
2724     push @$cmd, '-daemonize';
2725
2726     if ($conf->{smbios1}) {
2727         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2728     }
2729
2730     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2731         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2732         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2733         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2734         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2735         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2736         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2737     }
2738
2739     if ($q35) {
2740         # the q35 chipset support native usb2, so we enable usb controller
2741         # by default for this machine type
2742         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2743     } else {
2744         $pciaddr = print_pci_addr("piix3", $bridges);
2745         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2746
2747         my $use_usb2 = 0;
2748         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2749             next if !$conf->{"usb$i"};
2750             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2751             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2752             $use_usb2 = 1;
2753         }
2754         # include usb device config
2755         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2756     }
2757
2758     # add usb3 controller if needed
2759
2760     my $use_usb3 = 0;
2761     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2762         next if !$conf->{"usb$i"};
2763         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2764         next if !$d || !$d->{usb3};
2765         $use_usb3 = 1;
2766     }
2767
2768     $pciaddr = print_pci_addr("xhci", $bridges);
2769     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2770
2771     my $vga = $conf->{vga};
2772
2773     my $qxlnum = vga_conf_has_spice($vga);
2774     $vga = 'qxl' if $qxlnum;
2775
2776     if (!$vga) {
2777         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2778                                 $conf->{ostype} eq 'win7' ||
2779                                 $conf->{ostype} eq 'w2k8')) {
2780             $vga = 'std';
2781         } else {
2782             $vga = 'cirrus';
2783         }
2784     }
2785
2786     # enable absolute mouse coordinates (needed by vnc)
2787     my $tablet;
2788     if (defined($conf->{tablet})) {
2789         $tablet = $conf->{tablet};
2790     } else {
2791         $tablet = $defaults->{tablet};
2792         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2793         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2794     }
2795
2796     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2797
2798     my $kvm_off = 0;
2799     # host pci devices
2800     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2801         my $d = parse_hostpci($conf->{"hostpci$i"});
2802         next if !$d;
2803
2804         my $pcie = $d->{pcie};
2805         if($pcie){
2806             die "q35 machine model is not enabled" if !$q35;
2807             $pciaddr = print_pcie_addr("hostpci$i");
2808         }else{
2809             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2810         }
2811
2812         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2813         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2814         if ($xvga && $xvga ne '') {
2815             $kvm_off = 1;
2816             $vga = 'none';
2817             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2818                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2819             }
2820             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2821                 $xvga = "";
2822             }
2823         }
2824         my $pcidevices = $d->{pciid};
2825         my $multifunction = 1 if @$pcidevices > 1;
2826
2827         my $j=0;
2828         foreach my $pcidevice (@$pcidevices) {
2829
2830             my $id = "hostpci$i";
2831             $id .= ".$j" if $multifunction;
2832             my $addr = $pciaddr;
2833             $addr .= ".$j" if $multifunction;
2834             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2835
2836             if($j == 0){
2837                 $devicestr .= "$rombar$xvga";
2838                 $devicestr .= ",multifunction=on" if $multifunction;
2839             }
2840
2841             push @$devices, '-device', $devicestr;
2842             $j++;
2843         }
2844     }
2845
2846     # usb devices
2847     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2848         next if !$conf->{"usb$i"};
2849         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2850         next if !$d;
2851
2852         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2853         my $usbbus = '';
2854         if (defined($d->{usb3}) && $d->{usb3}) {
2855             $usbbus = ',bus=xhci.0';
2856         }
2857
2858         if (defined($d->{host})) {
2859             $d = parse_usb_device($d->{host});
2860             if (defined($d->{vendorid}) && defined($d->{productid})) {
2861                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2862             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2863                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2864             } elsif (defined($d->{spice}) && $d->{spice}) {
2865                 # usb redir support for spice, currently no usb3
2866                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2867                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2868             }
2869         }
2870     }
2871
2872     # serial devices
2873     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2874         if (my $path = $conf->{"serial$i"}) {
2875             if ($path eq 'socket') {
2876                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2877                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2878                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2879             } else {
2880                 die "no such serial device\n" if ! -c $path;
2881                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2882                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2883             }
2884         }
2885     }
2886
2887     # parallel devices
2888     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2889         if (my $path = $conf->{"parallel$i"}) {
2890             die "no such parallel device\n" if ! -c $path;
2891             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2892             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2893             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2894         }
2895     }
2896
2897     my $vmname = $conf->{name} || "vm$vmid";
2898
2899     push @$cmd, '-name', $vmname;
2900
2901     my $sockets = 1;
2902     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2903     $sockets = $conf->{sockets} if  $conf->{sockets};
2904
2905     my $cores = $conf->{cores} || 1;
2906
2907     my $maxcpus = $sockets * $cores;
2908
2909     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2910
2911     my $allowed_vcpus = $cpuinfo->{cpus};
2912
2913     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2914         if ($allowed_vcpus < $maxcpus);
2915
2916     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2917
2918     push @$cmd, '-nodefaults';
2919
2920     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2921
2922     my $bootindex_hash = {};
2923     my $i = 1;
2924     foreach my $o (split(//, $bootorder)) {
2925         $bootindex_hash->{$o} = $i*100;
2926         $i++;
2927     }
2928
2929     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2930
2931     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2932
2933     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2934
2935     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2936
2937     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2938         my $socket = vnc_socket($vmid);
2939         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2940     } else {
2941         push @$cmd, '-nographic';
2942     }
2943
2944     # time drift fix
2945     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2946
2947     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2948     my $useLocaltime = $conf->{localtime};
2949
2950     if ($ostype) {
2951         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2952
2953         if ($ostype =~ m/^w/) { # windows
2954             $useLocaltime = 1 if !defined($conf->{localtime});
2955
2956             # use time drift fix when acpi is enabled
2957             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2958                 $tdf = 1 if !defined($conf->{tdf});
2959             }
2960         }
2961
2962         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2963             $ostype eq 'wvista') {
2964             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2965             push @$cmd, '-no-hpet';
2966             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2967                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2968                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2969                 push @$cpuFlags , 'hv_time' if !$nokvm;
2970
2971             } else {
2972                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2973             }
2974         }
2975
2976         if ($ostype eq 'win7' || $ostype eq 'win8') {
2977             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2978         }
2979     }
2980
2981     push @$rtcFlags, 'driftfix=slew' if $tdf;
2982
2983     if ($nokvm) {
2984         push @$machineFlags, 'accel=tcg';
2985     } else {
2986         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2987     }
2988
2989     if ($machine_type) {
2990         push @$machineFlags, "type=${machine_type}";
2991     }
2992
2993     if ($conf->{startdate}) {
2994         push @$rtcFlags, "base=$conf->{startdate}";
2995     } elsif ($useLocaltime) {
2996         push @$rtcFlags, 'base=localtime';
2997     }
2998
2999     my $cpu = $nokvm ? "qemu64" : "kvm64";
3000     if (my $cputype = $conf->{cpu}) {
3001         my $cpuconf = PVE::JSONSchema::parse_property_string($cpudesc, $cputype)
3002             or die "Cannot parse cpu description: $cputype\n";
3003         $cpu = $cpuconf->{cputype};
3004         $kvm_off = 1 if $cpuconf->{hidden};
3005     }
3006
3007     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3008
3009     push @$cpuFlags , '-x2apic'
3010         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3011
3012     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3013
3014     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3015
3016     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3017
3018         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3019         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3020     }
3021
3022     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3023
3024     push @$cpuFlags, 'kvm=off' if $kvm_off;
3025
3026     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3027
3028     push @$cmd, '-cpu', $cpu;
3029
3030     my $memory = $conf->{memory} || $defaults->{memory};
3031     my $static_memory = 0;
3032     my $dimm_memory = 0;
3033
3034     if ($hotplug_features->{memory}) {
3035         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3036         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3037         $static_memory = $STATICMEM;
3038         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3039         $dimm_memory = $memory - $static_memory;
3040         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3041
3042     } else {
3043
3044         $static_memory = $memory;
3045         push @$cmd, '-m', $static_memory;
3046     }
3047
3048     if ($conf->{numa}) {
3049
3050         my $numa_totalmemory = undef;
3051         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3052             next if !$conf->{"numa$i"};
3053             my $numa = parse_numa($conf->{"numa$i"});
3054             next if !$numa;
3055             # memory
3056             die "missing numa node$i memory value\n" if !$numa->{memory};
3057             my $numa_memory = $numa->{memory};
3058             $numa_totalmemory += $numa_memory;
3059             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3060
3061             # cpus
3062             my $cpus_start = $numa->{cpus}->{start};
3063             die "missing numa node$i cpus\n" if !defined($cpus_start);
3064             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3065             my $cpus = $cpus_start;
3066             if (defined($cpus_end)) {
3067                 $cpus .= "-$cpus_end";
3068                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3069             }
3070
3071             # hostnodes
3072             my $hostnodes_start = $numa->{hostnodes}->{start};
3073             if (defined($hostnodes_start)) {
3074                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3075                 my $hostnodes = $hostnodes_start;
3076                 if (defined($hostnodes_end)) {
3077                     $hostnodes .= "-$hostnodes_end";
3078                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3079                 }
3080
3081                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3082                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3083                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3084                 }
3085
3086                 # policy
3087                 my $policy = $numa->{policy};
3088                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3089                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3090             }
3091
3092             push @$cmd, '-object', $numa_object;
3093             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3094         }
3095
3096         die "total memory for NUMA nodes must be equal to vm static memory\n"
3097             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3098
3099         #if no custom tology, we split memory and cores across numa nodes
3100         if(!$numa_totalmemory) {
3101
3102             my $numa_memory = ($static_memory / $sockets) . "M";
3103
3104             for (my $i = 0; $i < $sockets; $i++)  {
3105
3106                 my $cpustart = ($cores * $i);
3107                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3108                 my $cpus = $cpustart;
3109                 $cpus .= "-$cpuend" if $cpuend;
3110
3111                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3112                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3113             }
3114         }
3115     }
3116
3117     if ($hotplug_features->{memory}) {
3118         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3119             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3120             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3121             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3122
3123             #if dimm_memory is not aligned to dimm map
3124             if($current_size > $memory) {
3125                  $conf->{memory} = $current_size;
3126                  PVE::QemuConfig->write_config($vmid, $conf);
3127             }
3128         });
3129     }
3130
3131     push @$cmd, '-S' if $conf->{freeze};
3132
3133     # set keyboard layout
3134     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3135     push @$cmd, '-k', $kb if $kb;
3136
3137     # enable sound
3138     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3139     #push @$cmd, '-soundhw', 'es1370';
3140     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3141
3142     if($conf->{agent}) {
3143         my $qgasocket = qmp_socket($vmid, 1);
3144         my $pciaddr = print_pci_addr("qga0", $bridges);
3145         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3146         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3147         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3148     }
3149
3150     my $spice_port;
3151
3152     if ($qxlnum) {
3153         if ($qxlnum > 1) {
3154             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3155                 for(my $i = 1; $i < $qxlnum; $i++){
3156                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3157                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3158                 }
3159             } else {
3160                 # assume other OS works like Linux
3161                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3162                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3163             }
3164         }
3165
3166         my $pciaddr = print_pci_addr("spice", $bridges);
3167
3168         my $nodename = PVE::INotify::nodename();
3169         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3170         $spice_port = PVE::Tools::next_spice_port($pfamily);
3171
3172         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3173
3174         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3175         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3176         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3177     }
3178
3179     # enable balloon by default, unless explicitly disabled
3180     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3181         $pciaddr = print_pci_addr("balloon0", $bridges);
3182         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3183     }
3184
3185     if ($conf->{watchdog}) {
3186         my $wdopts = parse_watchdog($conf->{watchdog});
3187         $pciaddr = print_pci_addr("watchdog", $bridges);
3188         my $watchdog = $wdopts->{model} || 'i6300esb';
3189         push @$devices, '-device', "$watchdog$pciaddr";
3190         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3191     }
3192
3193     my $vollist = [];
3194     my $scsicontroller = {};
3195     my $ahcicontroller = {};
3196     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3197
3198     # Add iscsi initiator name if available
3199     if (my $initiator = get_initiator_name()) {
3200         push @$devices, '-iscsi', "initiator-name=$initiator";
3201     }
3202
3203     foreach_drive($conf, sub {
3204         my ($ds, $drive) = @_;
3205
3206         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3207             push @$vollist, $drive->{file};
3208         }
3209
3210         $use_virtio = 1 if $ds =~ m/^virtio/;
3211
3212         if (drive_is_cdrom ($drive)) {
3213             if ($bootindex_hash->{d}) {
3214                 $drive->{bootindex} = $bootindex_hash->{d};
3215                 $bootindex_hash->{d} += 1;
3216             }
3217         } else {
3218             if ($bootindex_hash->{c}) {
3219                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3220                 $bootindex_hash->{c} += 1;
3221             }
3222         }
3223
3224         if($drive->{interface} eq 'virtio'){
3225            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3226         }
3227
3228         if ($drive->{interface} eq 'scsi') {
3229
3230             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3231
3232             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3233             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3234
3235             my $iothread = '';
3236             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3237                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3238                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3239             }
3240
3241             my $queues = '';
3242             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3243                 $queues = ",num_queues=$drive->{queues}";
3244             }
3245
3246             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3247             $scsicontroller->{$controller}=1;
3248         }
3249
3250         if ($drive->{interface} eq 'sata') {
3251            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3252            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3253            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3254            $ahcicontroller->{$controller}=1;
3255         }
3256
3257         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3258         push @$devices, '-drive',$drive_cmd;
3259         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3260     });
3261
3262     for (my $i = 0; $i < $MAX_NETS; $i++) {
3263          next if !$conf->{"net$i"};
3264          my $d = parse_net($conf->{"net$i"});
3265          next if !$d;
3266
3267          $use_virtio = 1 if $d->{model} eq 'virtio';
3268
3269          if ($bootindex_hash->{n}) {
3270             $d->{bootindex} = $bootindex_hash->{n};
3271             $bootindex_hash->{n} += 1;
3272          }
3273
3274          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3275          push @$devices, '-netdev', $netdevfull;
3276
3277          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3278          push @$devices, '-device', $netdevicefull;
3279     }
3280
3281     if (!$q35) {
3282         # add pci bridges
3283         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3284            $bridges->{1} = 1;
3285            $bridges->{2} = 1;
3286         }
3287
3288         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3289
3290         while (my ($k, $v) = each %$bridges) {
3291             $pciaddr = print_pci_addr("pci.$k");
3292             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3293         }
3294     }
3295
3296     # add custom args
3297     if ($conf->{args}) {
3298         my $aa = PVE::Tools::split_args($conf->{args});
3299         push @$cmd, @$aa;
3300     }
3301
3302     push @$cmd, @$devices;
3303     push @$cmd, '-rtc', join(',', @$rtcFlags)
3304         if scalar(@$rtcFlags);
3305     push @$cmd, '-machine', join(',', @$machineFlags)
3306         if scalar(@$machineFlags);
3307     push @$cmd, '-global', join(',', @$globalFlags)
3308         if scalar(@$globalFlags);
3309
3310     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3311 }
3312
3313 sub vnc_socket {
3314     my ($vmid) = @_;
3315     return "${var_run_tmpdir}/$vmid.vnc";
3316 }
3317
3318 sub spice_port {
3319     my ($vmid) = @_;
3320
3321     my $res = vm_mon_cmd($vmid, 'query-spice');
3322
3323     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3324 }
3325
3326 sub qmp_socket {
3327     my ($vmid, $qga) = @_;
3328     my $sockettype = $qga ? 'qga' : 'qmp';
3329     return "${var_run_tmpdir}/$vmid.$sockettype";
3330 }
3331
3332 sub pidfile_name {
3333     my ($vmid) = @_;
3334     return "${var_run_tmpdir}/$vmid.pid";
3335 }
3336
3337 sub vm_devices_list {
3338     my ($vmid) = @_;
3339
3340     my $res = vm_mon_cmd($vmid, 'query-pci');
3341     my $devices = {};
3342     foreach my $pcibus (@$res) {
3343         foreach my $device (@{$pcibus->{devices}}) {
3344             next if !$device->{'qdev_id'};
3345             if ($device->{'pci_bridge'}) {
3346                 $devices->{$device->{'qdev_id'}} = 1;
3347                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3348                     next if !$bridge_device->{'qdev_id'};
3349                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3350                     $devices->{$device->{'qdev_id'}}++;
3351                 }
3352             } else {
3353                 $devices->{$device->{'qdev_id'}} = 1;
3354             }
3355         }
3356     }
3357
3358     my $resblock = vm_mon_cmd($vmid, 'query-block');
3359     foreach my $block (@$resblock) {
3360         if($block->{device} =~ m/^drive-(\S+)/){
3361                 $devices->{$1} = 1;
3362         }
3363     }
3364
3365     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3366     foreach my $mice (@$resmice) {
3367         if ($mice->{name} eq 'QEMU HID Tablet') {
3368             $devices->{tablet} = 1;
3369             last;
3370         }
3371     }
3372
3373     return $devices;
3374 }
3375
3376 sub vm_deviceplug {
3377     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3378
3379     my $q35 = machine_type_is_q35($conf);
3380
3381     my $devices_list = vm_devices_list($vmid);
3382     return 1 if defined($devices_list->{$deviceid});
3383
3384     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3385
3386     if ($deviceid eq 'tablet') {
3387
3388         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3389
3390     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3391
3392         qemu_iothread_add($vmid, $deviceid, $device);
3393
3394         qemu_driveadd($storecfg, $vmid, $device);
3395         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3396
3397         qemu_deviceadd($vmid, $devicefull);
3398         eval { qemu_deviceaddverify($vmid, $deviceid); };
3399         if (my $err = $@) {
3400             eval { qemu_drivedel($vmid, $deviceid); };
3401             warn $@ if $@;
3402             die $err;
3403         }
3404
3405     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3406
3407
3408         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3409         my $pciaddr = print_pci_addr($deviceid);
3410         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3411
3412         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3413
3414         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3415             qemu_iothread_add($vmid, $deviceid, $device);
3416             $devicefull .= ",iothread=iothread-$deviceid";
3417         }
3418
3419         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3420             $devicefull .= ",num_queues=$device->{queues}";
3421         }
3422
3423         qemu_deviceadd($vmid, $devicefull);
3424         qemu_deviceaddverify($vmid, $deviceid);
3425
3426     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3427
3428         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3429         qemu_driveadd($storecfg, $vmid, $device);
3430
3431         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3432         eval { qemu_deviceadd($vmid, $devicefull); };
3433         if (my $err = $@) {
3434             eval { qemu_drivedel($vmid, $deviceid); };
3435             warn $@ if $@;
3436             die $err;
3437         }
3438
3439     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3440
3441         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3442
3443         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3444         my $use_old_bios_files = undef;
3445         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3446
3447         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3448         qemu_deviceadd($vmid, $netdevicefull);
3449         eval { qemu_deviceaddverify($vmid, $deviceid); };
3450         if (my $err = $@) {
3451             eval { qemu_netdevdel($vmid, $deviceid); };
3452             warn $@ if $@;
3453             die $err;
3454         }
3455
3456     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3457
3458         my $bridgeid = $2;
3459         my $pciaddr = print_pci_addr($deviceid);
3460         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3461
3462         qemu_deviceadd($vmid, $devicefull);
3463         qemu_deviceaddverify($vmid, $deviceid);
3464
3465     } else {
3466         die "can't hotplug device '$deviceid'\n";
3467     }
3468
3469     return 1;
3470 }
3471
3472 # fixme: this should raise exceptions on error!
3473 sub vm_deviceunplug {
3474     my ($vmid, $conf, $deviceid) = @_;
3475
3476     my $devices_list = vm_devices_list($vmid);
3477     return 1 if !defined($devices_list->{$deviceid});
3478
3479     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3480
3481     if ($deviceid eq 'tablet') {
3482
3483         qemu_devicedel($vmid, $deviceid);
3484
3485     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3486
3487         qemu_devicedel($vmid, $deviceid);
3488         qemu_devicedelverify($vmid, $deviceid);
3489         qemu_drivedel($vmid, $deviceid);
3490         qemu_iothread_del($conf, $vmid, $deviceid);
3491
3492     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3493
3494         qemu_devicedel($vmid, $deviceid);
3495         qemu_devicedelverify($vmid, $deviceid);
3496         qemu_iothread_del($conf, $vmid, $deviceid);
3497
3498     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3499
3500         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3501         my $device = parse_drive($deviceid, $conf->{$deviceid});
3502         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3503
3504         qemu_devicedel($vmid, $deviceid);
3505         qemu_drivedel($vmid, $deviceid);
3506         qemu_deletescsihw($conf, $vmid, $deviceid);
3507
3508     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3509
3510         qemu_devicedel($vmid, $deviceid);
3511         qemu_devicedelverify($vmid, $deviceid);
3512         qemu_netdevdel($vmid, $deviceid);
3513
3514     } else {
3515         die "can't unplug device '$deviceid'\n";
3516     }
3517
3518     return 1;
3519 }
3520
3521 sub qemu_deviceadd {
3522     my ($vmid, $devicefull) = @_;
3523
3524     $devicefull = "driver=".$devicefull;
3525     my %options =  split(/[=,]/, $devicefull);
3526
3527     vm_mon_cmd($vmid, "device_add" , %options);
3528 }
3529
3530 sub qemu_devicedel {
3531     my ($vmid, $deviceid) = @_;
3532
3533     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3534 }
3535
3536 sub qemu_iothread_add {
3537     my($vmid, $deviceid, $device) = @_;
3538
3539     if ($device->{iothread}) {
3540         my $iothreads = vm_iothreads_list($vmid);
3541         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3542     }
3543 }
3544
3545 sub qemu_iothread_del {
3546     my($conf, $vmid, $deviceid) = @_;
3547
3548     my $device = parse_drive($deviceid, $conf->{$deviceid});
3549     if ($device->{iothread}) {
3550         my $iothreads = vm_iothreads_list($vmid);
3551         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3552     }
3553 }
3554
3555 sub qemu_objectadd {
3556     my($vmid, $objectid, $qomtype) = @_;
3557
3558     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3559
3560     return 1;
3561 }
3562
3563 sub qemu_objectdel {
3564     my($vmid, $objectid) = @_;
3565
3566     vm_mon_cmd($vmid, "object-del", id => $objectid);
3567
3568     return 1;
3569 }
3570
3571 sub qemu_driveadd {
3572     my ($storecfg, $vmid, $device) = @_;
3573
3574     my $drive = print_drive_full($storecfg, $vmid, $device);
3575     $drive =~ s/\\/\\\\/g;
3576     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3577
3578     # If the command succeeds qemu prints: "OK"
3579     return 1 if $ret =~ m/OK/s;
3580
3581     die "adding drive failed: $ret\n";
3582 }
3583
3584 sub qemu_drivedel {
3585     my($vmid, $deviceid) = @_;
3586
3587     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3588     $ret =~ s/^\s+//;
3589
3590     return 1 if $ret eq "";
3591
3592     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3593     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3594
3595     die "deleting drive $deviceid failed : $ret\n";
3596 }
3597
3598 sub qemu_deviceaddverify {
3599     my ($vmid, $deviceid) = @_;
3600
3601     for (my $i = 0; $i <= 5; $i++) {
3602          my $devices_list = vm_devices_list($vmid);
3603          return 1 if defined($devices_list->{$deviceid});
3604          sleep 1;
3605     }
3606
3607     die "error on hotplug device '$deviceid'\n";
3608 }
3609
3610
3611 sub qemu_devicedelverify {
3612     my ($vmid, $deviceid) = @_;
3613
3614     # need to verify that the device is correctly removed as device_del
3615     # is async and empty return is not reliable
3616
3617     for (my $i = 0; $i <= 5; $i++) {
3618          my $devices_list = vm_devices_list($vmid);
3619          return 1 if !defined($devices_list->{$deviceid});
3620          sleep 1;
3621     }
3622
3623     die "error on hot-unplugging device '$deviceid'\n";
3624 }
3625
3626 sub qemu_findorcreatescsihw {
3627     my ($storecfg, $conf, $vmid, $device) = @_;
3628
3629     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3630
3631     my $scsihwid="$controller_prefix$controller";
3632     my $devices_list = vm_devices_list($vmid);
3633
3634     if(!defined($devices_list->{$scsihwid})) {
3635         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3636     }
3637
3638     return 1;
3639 }
3640
3641 sub qemu_deletescsihw {
3642     my ($conf, $vmid, $opt) = @_;
3643
3644     my $device = parse_drive($opt, $conf->{$opt});
3645
3646     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3647         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3648         return 1;
3649     }
3650
3651     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3652
3653     my $devices_list = vm_devices_list($vmid);
3654     foreach my $opt (keys %{$devices_list}) {
3655         if (PVE::QemuServer::is_valid_drivename($opt)) {
3656             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3657             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3658                 return 1;
3659             }
3660         }
3661     }
3662
3663     my $scsihwid="scsihw$controller";
3664
3665     vm_deviceunplug($vmid, $conf, $scsihwid);
3666
3667     return 1;
3668 }
3669
3670 sub qemu_add_pci_bridge {
3671     my ($storecfg, $conf, $vmid, $device) = @_;
3672
3673     my $bridges = {};
3674
3675     my $bridgeid;
3676
3677     print_pci_addr($device, $bridges);
3678
3679     while (my ($k, $v) = each %$bridges) {
3680         $bridgeid = $k;
3681     }
3682     return 1 if !defined($bridgeid) || $bridgeid < 1;
3683
3684     my $bridge = "pci.$bridgeid";
3685     my $devices_list = vm_devices_list($vmid);
3686
3687     if (!defined($devices_list->{$bridge})) {
3688         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3689     }
3690
3691     return 1;
3692 }
3693
3694 sub qemu_set_link_status {
3695     my ($vmid, $device, $up) = @_;
3696
3697     vm_mon_cmd($vmid, "set_link", name => $device,
3698                up => $up ? JSON::true : JSON::false);
3699 }
3700
3701 sub qemu_netdevadd {
3702     my ($vmid, $conf, $device, $deviceid) = @_;
3703
3704     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3705     my %options =  split(/[=,]/, $netdev);
3706
3707     vm_mon_cmd($vmid, "netdev_add",  %options);
3708     return 1;
3709 }
3710
3711 sub qemu_netdevdel {
3712     my ($vmid, $deviceid) = @_;
3713
3714     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3715 }
3716
3717 sub qemu_cpu_hotplug {
3718     my ($vmid, $conf, $vcpus) = @_;
3719
3720     my $sockets = 1;
3721     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3722     $sockets = $conf->{sockets} if  $conf->{sockets};
3723     my $cores = $conf->{cores} || 1;
3724     my $maxcpus = $sockets * $cores;
3725
3726     $vcpus = $maxcpus if !$vcpus;
3727
3728     die "you can't add more vcpus than maxcpus\n"
3729         if $vcpus > $maxcpus;
3730
3731     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3732     die "online cpu unplug is not yet possible\n"
3733         if $vcpus < $currentvcpus;
3734
3735     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3736     die "vcpus in running vm is different than configuration\n"
3737         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3738
3739     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3740         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3741     }
3742 }
3743
3744 sub qemu_memory_hotplug {
3745     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3746
3747     return $value if !check_running($vmid);
3748
3749     my $memory = $conf->{memory} || $defaults->{memory};
3750     $value = $defaults->{memory} if !$value;
3751     return $value if $value == $memory;
3752
3753     my $static_memory = $STATICMEM;
3754     my $dimm_memory = $memory - $static_memory;
3755
3756     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3757     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3758
3759
3760     my $sockets = 1;
3761     $sockets = $conf->{sockets} if $conf->{sockets};
3762
3763     if($value > $memory) {
3764
3765         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3766             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3767
3768                 return if $current_size <= $conf->{memory};
3769
3770                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3771                 if (my $err = $@) {
3772                     eval { qemu_objectdel($vmid, "mem-$name"); };
3773                     die $err;
3774                 }
3775
3776                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3777                 if (my $err = $@) {
3778                     eval { qemu_objectdel($vmid, "mem-$name"); };
3779                     die $err;
3780                 }
3781                 #update conf after each succesful module hotplug
3782                 $conf->{memory} = $current_size;
3783                 PVE::QemuConfig->write_config($vmid, $conf);
3784         });
3785
3786     } else {
3787
3788         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3789             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3790
3791                 return if $current_size >= $conf->{memory};
3792                 print "try to unplug memory dimm $name\n";
3793
3794                 my $retry = 0;
3795                 while (1) {
3796                     eval { qemu_devicedel($vmid, $name) };
3797                     sleep 3;
3798                     my $dimm_list = qemu_dimm_list($vmid);
3799                     last if !$dimm_list->{$name};
3800                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3801                     $retry++;
3802                 }
3803
3804                 #update conf after each succesful module unplug
3805                 $conf->{memory} = $current_size;
3806
3807                 eval { qemu_objectdel($vmid, "mem-$name"); };
3808                 PVE::QemuConfig->write_config($vmid, $conf);
3809         });
3810     }
3811 }
3812
3813 sub qemu_dimm_list {
3814     my ($vmid) = @_;
3815
3816     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3817     my $dimms = {};
3818
3819     foreach my $dimm (@$dimmarray) {
3820
3821         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3822         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3823         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3824         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3825         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3826     }
3827     return $dimms;
3828 }
3829
3830 sub qemu_block_set_io_throttle {
3831     my ($vmid, $deviceid,
3832         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3833         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3834
3835     return if !check_running($vmid) ;
3836
3837     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3838         bps => int($bps),
3839         bps_rd => int($bps_rd),
3840         bps_wr => int($bps_wr),
3841         iops => int($iops),
3842         iops_rd => int($iops_rd),
3843         iops_wr => int($iops_wr),
3844         bps_max => int($bps_max),
3845         bps_rd_max => int($bps_rd_max),
3846         bps_wr_max => int($bps_wr_max),
3847         iops_max => int($iops_max),
3848         iops_rd_max => int($iops_rd_max),
3849         iops_wr_max => int($iops_wr_max)
3850     );
3851
3852 }
3853
3854 # old code, only used to shutdown old VM after update
3855 sub __read_avail {
3856     my ($fh, $timeout) = @_;
3857
3858     my $sel = new IO::Select;
3859     $sel->add($fh);
3860
3861     my $res = '';
3862     my $buf;
3863
3864     my @ready;
3865     while (scalar (@ready = $sel->can_read($timeout))) {
3866         my $count;
3867         if ($count = $fh->sysread($buf, 8192)) {
3868             if ($buf =~ /^(.*)\(qemu\) $/s) {
3869                 $res .= $1;
3870                 last;
3871             } else {
3872                 $res .= $buf;
3873             }
3874         } else {
3875             if (!defined($count)) {
3876                 die "$!\n";
3877             }
3878             last;
3879         }
3880     }
3881
3882     die "monitor read timeout\n" if !scalar(@ready);
3883
3884     return $res;
3885 }
3886
3887 # old code, only used to shutdown old VM after update
3888 sub vm_monitor_command {
3889     my ($vmid, $cmdstr, $nocheck) = @_;
3890
3891     my $res;
3892
3893     eval {
3894         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3895
3896         my $sname = "${var_run_tmpdir}/$vmid.mon";
3897
3898         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3899             die "unable to connect to VM $vmid socket - $!\n";
3900
3901         my $timeout = 3;
3902
3903         # hack: migrate sometime blocks the monitor (when migrate_downtime
3904         # is set)
3905         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3906             $timeout = 60*60; # 1 hour
3907         }
3908
3909         # read banner;
3910         my $data = __read_avail($sock, $timeout);
3911
3912         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3913             die "got unexpected qemu monitor banner\n";
3914         }
3915
3916         my $sel = new IO::Select;
3917         $sel->add($sock);
3918
3919         if (!scalar(my @ready = $sel->can_write($timeout))) {
3920             die "monitor write error - timeout";
3921         }
3922
3923         my $fullcmd = "$cmdstr\r";
3924
3925         # syslog('info', "VM $vmid monitor command: $cmdstr");
3926
3927         my $b;
3928         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3929             die "monitor write error - $!";
3930         }
3931
3932         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3933
3934         $timeout = 20;
3935
3936         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3937             $timeout = 60*60; # 1 hour
3938         } elsif ($cmdstr =~ m/^(eject|change)/) {
3939             $timeout = 60; # note: cdrom mount command is slow
3940         }
3941         if ($res = __read_avail($sock, $timeout)) {
3942
3943             my @lines = split("\r?\n", $res);
3944
3945             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3946
3947             $res = join("\n", @lines);
3948             $res .= "\n";
3949         }
3950     };
3951
3952     my $err = $@;
3953
3954     if ($err) {
3955         syslog("err", "VM $vmid monitor command failed - $err");
3956         die $err;
3957     }
3958
3959     return $res;
3960 }
3961
3962 sub qemu_block_resize {
3963     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3964
3965     my $running = check_running($vmid);
3966
3967     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3968
3969     return if !$running;
3970
3971     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3972
3973 }
3974
3975 sub qemu_volume_snapshot {
3976     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3977
3978     my $running = check_running($vmid);
3979
3980     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3981         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3982     } else {
3983         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3984     }
3985 }
3986
3987 sub qemu_volume_snapshot_delete {
3988     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3989
3990     my $running = check_running($vmid);
3991
3992     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3993
3994     return if !$running;
3995
3996     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3997 }
3998
3999 sub set_migration_caps {
4000     my ($vmid) = @_;
4001
4002     my $cap_ref = [];
4003
4004     my $enabled_cap = {
4005         "auto-converge" => 1,
4006         "xbzrle" => 1,
4007         "x-rdma-pin-all" => 0,
4008         "zero-blocks" => 0,
4009         "compress" => 0
4010     };
4011
4012     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4013
4014     for my $supported_capability (@$supported_capabilities) {
4015         push @$cap_ref, {
4016             capability => $supported_capability->{capability},
4017             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4018         };
4019     }
4020
4021     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4022 }
4023
4024 my $fast_plug_option = {
4025     'lock' => 1,
4026     'name' => 1,
4027     'onboot' => 1,
4028     'shares' => 1,
4029     'startup' => 1,
4030     'description' => 1,
4031 };
4032
4033 # hotplug changes in [PENDING]
4034 # $selection hash can be used to only apply specified options, for
4035 # example: { cores => 1 } (only apply changed 'cores')
4036 # $errors ref is used to return error messages
4037 sub vmconfig_hotplug_pending {
4038     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4039
4040     my $defaults = load_defaults();
4041
4042     # commit values which do not have any impact on running VM first
4043     # Note: those option cannot raise errors, we we do not care about
4044     # $selection and always apply them.
4045
4046     my $add_error = sub {
4047         my ($opt, $msg) = @_;
4048         $errors->{$opt} = "hotplug problem - $msg";
4049     };
4050
4051     my $changes = 0;
4052     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4053         if ($fast_plug_option->{$opt}) {
4054             $conf->{$opt} = $conf->{pending}->{$opt};
4055             delete $conf->{pending}->{$opt};
4056             $changes = 1;
4057         }
4058     }
4059
4060     if ($changes) {
4061         PVE::QemuConfig->write_config($vmid, $conf);
4062         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4063     }
4064
4065     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4066
4067     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4068     while (my ($opt, $force) = each %$pending_delete_hash) {
4069         next if $selection && !$selection->{$opt};
4070         eval {
4071             if ($opt eq 'hotplug') {
4072                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4073             } elsif ($opt eq 'tablet') {
4074                 die "skip\n" if !$hotplug_features->{usb};
4075                 if ($defaults->{tablet}) {
4076                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4077                 } else {
4078                     vm_deviceunplug($vmid, $conf, $opt);
4079                 }
4080             } elsif ($opt eq 'vcpus') {
4081                 die "skip\n" if !$hotplug_features->{cpu};
4082                 qemu_cpu_hotplug($vmid, $conf, undef);
4083             } elsif ($opt eq 'balloon') {
4084                 # enable balloon device is not hotpluggable
4085                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4086             } elsif ($fast_plug_option->{$opt}) {
4087                 # do nothing
4088             } elsif ($opt =~ m/^net(\d+)$/) {
4089                 die "skip\n" if !$hotplug_features->{network};
4090                 vm_deviceunplug($vmid, $conf, $opt);
4091             } elsif (is_valid_drivename($opt)) {
4092                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4093                 vm_deviceunplug($vmid, $conf, $opt);
4094                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4095             } elsif ($opt =~ m/^memory$/) {
4096                 die "skip\n" if !$hotplug_features->{memory};
4097                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4098             } elsif ($opt eq 'cpuunits') {
4099                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4100             } elsif ($opt eq 'cpulimit') {
4101                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4102             } else {
4103                 die "skip\n";
4104             }
4105         };
4106         if (my $err = $@) {
4107             &$add_error($opt, $err) if $err ne "skip\n";
4108         } else {
4109             # save new config if hotplug was successful
4110             delete $conf->{$opt};
4111             vmconfig_undelete_pending_option($conf, $opt);
4112             PVE::QemuConfig->write_config($vmid, $conf);
4113             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4114         }
4115     }
4116
4117     foreach my $opt (keys %{$conf->{pending}}) {
4118         next if $selection && !$selection->{$opt};
4119         my $value = $conf->{pending}->{$opt};
4120         eval {
4121             if ($opt eq 'hotplug') {
4122                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4123             } elsif ($opt eq 'tablet') {
4124                 die "skip\n" if !$hotplug_features->{usb};
4125                 if ($value == 1) {
4126                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4127                 } elsif ($value == 0) {
4128                     vm_deviceunplug($vmid, $conf, $opt);
4129                 }
4130             } elsif ($opt eq 'vcpus') {
4131                 die "skip\n" if !$hotplug_features->{cpu};
4132                 qemu_cpu_hotplug($vmid, $conf, $value);
4133             } elsif ($opt eq 'balloon') {
4134                 # enable/disable balloning device is not hotpluggable
4135                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4136                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4137                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4138
4139                 # allow manual ballooning if shares is set to zero
4140                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4141                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4142                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4143                 }
4144             } elsif ($opt =~ m/^net(\d+)$/) {
4145                 # some changes can be done without hotplug
4146                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4147                                     $vmid, $opt, $value);
4148             } elsif (is_valid_drivename($opt)) {
4149                 # some changes can be done without hotplug
4150                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4151                                      $vmid, $opt, $value, 1);
4152             } elsif ($opt =~ m/^memory$/) { #dimms
4153                 die "skip\n" if !$hotplug_features->{memory};
4154                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4155             } elsif ($opt eq 'cpuunits') {
4156                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4157             } elsif ($opt eq 'cpulimit') {
4158                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4159                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4160             } else {
4161                 die "skip\n";  # skip non-hot-pluggable options
4162             }
4163         };
4164         if (my $err = $@) {
4165             &$add_error($opt, $err) if $err ne "skip\n";
4166         } else {
4167             # save new config if hotplug was successful
4168             $conf->{$opt} = $value;
4169             delete $conf->{pending}->{$opt};
4170             PVE::QemuConfig->write_config($vmid, $conf);
4171             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4172         }
4173     }
4174 }
4175
4176 sub try_deallocate_drive {
4177     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4178
4179     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4180         my $volid = $drive->{file};
4181         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4182             my $sid = PVE::Storage::parse_volume_id($volid);
4183             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4184
4185             # check if the disk is really unused
4186             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4187                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4188             PVE::Storage::vdisk_free($storecfg, $volid);
4189             return 1;
4190         } else {
4191             # If vm is not owner of this disk remove from config
4192             return 1;
4193         }
4194     }
4195
4196     return undef;
4197 }
4198
4199 sub vmconfig_delete_or_detach_drive {
4200     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4201
4202     my $drive = parse_drive($opt, $conf->{$opt});
4203
4204     my $rpcenv = PVE::RPCEnvironment::get();
4205     my $authuser = $rpcenv->get_user();
4206
4207     if ($force) {
4208         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4209         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4210     } else {
4211         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4212     }
4213 }
4214
4215 sub vmconfig_apply_pending {
4216     my ($vmid, $conf, $storecfg) = @_;
4217
4218     # cold plug
4219
4220     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4221     while (my ($opt, $force) = each %$pending_delete_hash) {
4222         die "internal error" if $opt =~ m/^unused/;
4223         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4224         if (!defined($conf->{$opt})) {
4225             vmconfig_undelete_pending_option($conf, $opt);
4226             PVE::QemuConfig->write_config($vmid, $conf);
4227         } elsif (is_valid_drivename($opt)) {
4228             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4229             vmconfig_undelete_pending_option($conf, $opt);
4230             delete $conf->{$opt};
4231             PVE::QemuConfig->write_config($vmid, $conf);
4232         } else {
4233             vmconfig_undelete_pending_option($conf, $opt);
4234             delete $conf->{$opt};
4235             PVE::QemuConfig->write_config($vmid, $conf);
4236         }
4237     }
4238
4239     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4240
4241     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4242         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4243
4244         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4245             # skip if nothing changed
4246         } elsif (is_valid_drivename($opt)) {
4247             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4248                 if defined($conf->{$opt});
4249             $conf->{$opt} = $conf->{pending}->{$opt};
4250         } else {
4251             $conf->{$opt} = $conf->{pending}->{$opt};
4252         }
4253
4254         delete $conf->{pending}->{$opt};
4255         PVE::QemuConfig->write_config($vmid, $conf);
4256     }
4257 }
4258
4259 my $safe_num_ne = sub {
4260     my ($a, $b) = @_;
4261
4262     return 0 if !defined($a) && !defined($b);
4263     return 1 if !defined($a);
4264     return 1 if !defined($b);
4265
4266     return $a != $b;
4267 };
4268
4269 my $safe_string_ne = sub {
4270     my ($a, $b) = @_;
4271
4272     return 0 if !defined($a) && !defined($b);
4273     return 1 if !defined($a);
4274     return 1 if !defined($b);
4275
4276     return $a ne $b;
4277 };
4278
4279 sub vmconfig_update_net {
4280     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4281
4282     my $newnet = parse_net($value);
4283
4284     if ($conf->{$opt}) {
4285         my $oldnet = parse_net($conf->{$opt});
4286
4287         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4288             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4289             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4290             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4291
4292             # for non online change, we try to hot-unplug
4293             die "skip\n" if !$hotplug;
4294             vm_deviceunplug($vmid, $conf, $opt);
4295         } else {
4296
4297             die "internal error" if $opt !~ m/net(\d+)/;
4298             my $iface = "tap${vmid}i$1";
4299
4300             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4301                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4302                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4303                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4304                 PVE::Network::tap_unplug($iface);
4305                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4306             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4307                 # Rate can be applied on its own but any change above needs to
4308                 # include the rate in tap_plug since OVS resets everything.
4309                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4310             }
4311
4312             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4313                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4314             }
4315
4316             return 1;
4317         }
4318     }
4319
4320     if ($hotplug) {
4321         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4322     } else {
4323         die "skip\n";
4324     }
4325 }
4326
4327 sub vmconfig_update_disk {
4328     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4329
4330     # fixme: do we need force?
4331
4332     my $drive = parse_drive($opt, $value);
4333
4334     if ($conf->{$opt}) {
4335
4336         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4337
4338             my $media = $drive->{media} || 'disk';
4339             my $oldmedia = $old_drive->{media} || 'disk';
4340             die "unable to change media type\n" if $media ne $oldmedia;
4341
4342             if (!drive_is_cdrom($old_drive)) {
4343
4344                 if ($drive->{file} ne $old_drive->{file}) {
4345
4346                     die "skip\n" if !$hotplug;
4347
4348                     # unplug and register as unused
4349                     vm_deviceunplug($vmid, $conf, $opt);
4350                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4351
4352                 } else {
4353                     # update existing disk
4354
4355                     # skip non hotpluggable value
4356                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4357                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4358                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4359                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4360                         die "skip\n";
4361                     }
4362
4363                     # apply throttle
4364                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4365                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4366                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4367                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4368                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4369                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4370                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4371                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4372                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4373                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4374                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4375                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4376
4377                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4378                                                    ($drive->{mbps} || 0)*1024*1024,
4379                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4380                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4381                                                    $drive->{iops} || 0,
4382                                                    $drive->{iops_rd} || 0,
4383                                                    $drive->{iops_wr} || 0,
4384                                                    ($drive->{mbps_max} || 0)*1024*1024,
4385                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4386                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4387                                                    $drive->{iops_max} || 0,
4388                                                    $drive->{iops_rd_max} || 0,
4389                                                    $drive->{iops_wr_max} || 0);
4390
4391                     }
4392
4393                     return 1;
4394                 }
4395
4396             } else { # cdrom
4397
4398                 if ($drive->{file} eq 'none') {
4399                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4400                 } else {
4401                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4402                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4403                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4404                 }
4405
4406                 return 1;
4407             }
4408         }
4409     }
4410
4411     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4412     # hotplug new disks
4413     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4414     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4415 }
4416
4417 sub vm_start {
4418     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4419         $forcemachine, $spice_ticket) = @_;
4420
4421     PVE::QemuConfig->lock_config($vmid, sub {
4422         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4423
4424         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4425
4426         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4427
4428         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4429
4430         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4431             vmconfig_apply_pending($vmid, $conf, $storecfg);
4432             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4433         }
4434
4435         my $defaults = load_defaults();
4436
4437         # set environment variable useful inside network script
4438         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4439
4440         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4441
4442         my $migrate_port = 0;
4443         my $migrate_uri;
4444         if ($statefile) {
4445             if ($statefile eq 'tcp') {
4446                 my $localip = "localhost";
4447                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4448                 my $nodename = PVE::INotify::nodename();
4449                 if ($datacenterconf->{migration_unsecure}) {
4450                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4451                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4452                 }
4453                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4454                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4455                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4456                 push @$cmd, '-incoming', $migrate_uri;
4457                 push @$cmd, '-S';
4458             } else {
4459                 push @$cmd, '-loadstate', $statefile;
4460             }
4461         } elsif ($paused) {
4462             push @$cmd, '-S';
4463         }
4464
4465         # host pci devices
4466         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4467           my $d = parse_hostpci($conf->{"hostpci$i"});
4468           next if !$d;
4469           my $pcidevices = $d->{pciid};
4470           foreach my $pcidevice (@$pcidevices) {
4471                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4472
4473                 my $info = pci_device_info("0000:$pciid");
4474                 die "IOMMU not present\n" if !check_iommu_support();
4475                 die "no pci device info for device '$pciid'\n" if !$info;
4476                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4477                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4478           }
4479         }
4480
4481         PVE::Storage::activate_volumes($storecfg, $vollist);
4482
4483         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4484                     umask => 0077); };
4485
4486         if (my $err = $@) {
4487             # deactivate volumes if start fails
4488             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4489             die "start failed: $err";
4490         }
4491
4492         print "migration listens on $migrate_uri\n" if $migrate_uri;
4493
4494         if ($statefile && $statefile ne 'tcp')  {
4495             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4496             warn $@ if $@;
4497         }
4498
4499         if ($migratedfrom) {
4500
4501             eval {
4502                 set_migration_caps($vmid);
4503             };
4504             warn $@ if $@;
4505
4506             if ($spice_port) {
4507                 print "spice listens on port $spice_port\n";
4508                 if ($spice_ticket) {
4509                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4510                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4511                 }
4512             }
4513
4514         } else {
4515
4516             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4517                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4518                     if $conf->{balloon};
4519             }
4520
4521             foreach my $opt (keys %$conf) {
4522                 next if $opt !~  m/^net\d+$/;
4523                 my $nicconf = parse_net($conf->{$opt});
4524                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4525             }
4526         }
4527
4528         vm_mon_cmd_nocheck($vmid, 'qom-set',
4529                     path => "machine/peripheral/balloon0",
4530                     property => "guest-stats-polling-interval",
4531                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4532
4533     });
4534 }
4535
4536 sub vm_mon_cmd {
4537     my ($vmid, $execute, %params) = @_;
4538
4539     my $cmd = { execute => $execute, arguments => \%params };
4540     vm_qmp_command($vmid, $cmd);
4541 }
4542
4543 sub vm_mon_cmd_nocheck {
4544     my ($vmid, $execute, %params) = @_;
4545
4546     my $cmd = { execute => $execute, arguments => \%params };
4547     vm_qmp_command($vmid, $cmd, 1);
4548 }
4549
4550 sub vm_qmp_command {
4551     my ($vmid, $cmd, $nocheck) = @_;
4552
4553     my $res;
4554
4555     my $timeout;
4556     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4557         $timeout = $cmd->{arguments}->{timeout};
4558         delete $cmd->{arguments}->{timeout};
4559     }
4560
4561     eval {
4562         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4563         my $sname = qmp_socket($vmid);
4564         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4565             my $qmpclient = PVE::QMPClient->new();
4566
4567             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4568         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4569             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4570                 if scalar(%{$cmd->{arguments}});
4571             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4572         } else {
4573             die "unable to open monitor socket\n";
4574         }
4575     };
4576     if (my $err = $@) {
4577         syslog("err", "VM $vmid qmp command failed - $err");
4578         die $err;
4579     }
4580
4581     return $res;
4582 }
4583
4584 sub vm_human_monitor_command {
4585     my ($vmid, $cmdline) = @_;
4586
4587     my $res;
4588
4589     my $cmd = {
4590         execute => 'human-monitor-command',
4591         arguments => { 'command-line' => $cmdline},
4592     };
4593
4594     return vm_qmp_command($vmid, $cmd);
4595 }
4596
4597 sub vm_commandline {
4598     my ($storecfg, $vmid) = @_;
4599
4600     my $conf = PVE::QemuConfig->load_config($vmid);
4601
4602     my $defaults = load_defaults();
4603
4604     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4605
4606     return join(' ', @$cmd);
4607 }
4608
4609 sub vm_reset {
4610     my ($vmid, $skiplock) = @_;
4611
4612     PVE::QemuConfig->lock_config($vmid, sub {
4613
4614         my $conf = PVE::QemuConfig->load_config($vmid);
4615
4616         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4617
4618         vm_mon_cmd($vmid, "system_reset");
4619     });
4620 }
4621
4622 sub get_vm_volumes {
4623     my ($conf) = @_;
4624
4625     my $vollist = [];
4626     foreach_volid($conf, sub {
4627         my ($volid, $is_cdrom) = @_;
4628
4629         return if $volid =~ m|^/|;
4630
4631         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4632         return if !$sid;
4633
4634         push @$vollist, $volid;
4635     });
4636
4637     return $vollist;
4638 }
4639
4640 sub vm_stop_cleanup {
4641     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4642
4643     eval {
4644
4645         if (!$keepActive) {
4646             my $vollist = get_vm_volumes($conf);
4647             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4648         }
4649
4650         foreach my $ext (qw(mon qmp pid vnc qga)) {
4651             unlink "/var/run/qemu-server/${vmid}.$ext";
4652         }
4653
4654         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4655     };
4656     warn $@ if $@; # avoid errors - just warn
4657 }
4658
4659 # Note: use $nockeck to skip tests if VM configuration file exists.
4660 # We need that when migration VMs to other nodes (files already moved)
4661 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4662 sub vm_stop {
4663     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4664
4665     $force = 1 if !defined($force) && !$shutdown;
4666
4667     if ($migratedfrom){
4668         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4669         kill 15, $pid if $pid;
4670         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4671         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4672         return;
4673     }
4674
4675     PVE::QemuConfig->lock_config($vmid, sub {
4676
4677         my $pid = check_running($vmid, $nocheck);
4678         return if !$pid;
4679
4680         my $conf;
4681         if (!$nocheck) {
4682             $conf = PVE::QemuConfig->load_config($vmid);
4683             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4684             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4685                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4686                 $timeout = $opts->{down} if $opts->{down};
4687             }
4688         }
4689
4690         $timeout = 60 if !defined($timeout);
4691
4692         eval {
4693             if ($shutdown) {
4694                 if (defined($conf) && $conf->{agent}) {
4695                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4696                 } else {
4697                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4698                 }
4699             } else {
4700                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4701             }
4702         };
4703         my $err = $@;
4704
4705         if (!$err) {
4706             my $count = 0;
4707             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4708                 $count++;
4709                 sleep 1;
4710             }
4711
4712             if ($count >= $timeout) {
4713                 if ($force) {
4714                     warn "VM still running - terminating now with SIGTERM\n";
4715                     kill 15, $pid;
4716                 } else {
4717                     die "VM quit/powerdown failed - got timeout\n";
4718                 }
4719             } else {
4720                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4721                 return;
4722             }
4723         } else {
4724             if ($force) {
4725                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4726                 kill 15, $pid;
4727             } else {
4728                 die "VM quit/powerdown failed\n";
4729             }
4730         }
4731
4732         # wait again
4733         $timeout = 10;
4734
4735         my $count = 0;
4736         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4737             $count++;
4738             sleep 1;
4739         }
4740
4741         if ($count >= $timeout) {
4742             warn "VM still running - terminating now with SIGKILL\n";
4743             kill 9, $pid;
4744             sleep 1;
4745         }
4746
4747         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4748    });
4749 }
4750
4751 sub vm_suspend {
4752     my ($vmid, $skiplock) = @_;
4753
4754     PVE::QemuConfig->lock_config($vmid, sub {
4755
4756         my $conf = PVE::QemuConfig->load_config($vmid);
4757
4758         PVE::QemuConfig->check_lock($conf)
4759             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4760
4761         vm_mon_cmd($vmid, "stop");
4762     });
4763 }
4764
4765 sub vm_resume {
4766     my ($vmid, $skiplock, $nocheck) = @_;
4767
4768     PVE::QemuConfig->lock_config($vmid, sub {
4769
4770         if (!$nocheck) {
4771
4772             my $conf = PVE::QemuConfig->load_config($vmid);
4773
4774             PVE::QemuConfig->check_lock($conf)
4775                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4776
4777             vm_mon_cmd($vmid, "cont");
4778
4779         } else {
4780             vm_mon_cmd_nocheck($vmid, "cont");
4781         }
4782     });
4783 }
4784
4785 sub vm_sendkey {
4786     my ($vmid, $skiplock, $key) = @_;
4787
4788     PVE::QemuConfig->lock_config($vmid, sub {
4789
4790         my $conf = PVE::QemuConfig->load_config($vmid);
4791
4792         # there is no qmp command, so we use the human monitor command
4793         vm_human_monitor_command($vmid, "sendkey $key");
4794     });
4795 }
4796
4797 sub vm_destroy {
4798     my ($storecfg, $vmid, $skiplock) = @_;
4799
4800     PVE::QemuConfig->lock_config($vmid, sub {
4801
4802         my $conf = PVE::QemuConfig->load_config($vmid);
4803
4804         if (!check_running($vmid)) {
4805             destroy_vm($storecfg, $vmid, undef, $skiplock);
4806         } else {
4807             die "VM $vmid is running - destroy failed\n";
4808         }
4809     });
4810 }
4811
4812 # pci helpers
4813
4814 sub file_write {
4815     my ($filename, $buf) = @_;
4816
4817     my $fh = IO::File->new($filename, "w");
4818     return undef if !$fh;
4819
4820     my $res = print $fh $buf;
4821
4822     $fh->close();
4823
4824     return $res;
4825 }
4826
4827 sub pci_device_info {
4828     my ($name) = @_;
4829
4830     my $res;
4831
4832     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4833     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4834
4835     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4836     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4837
4838     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4839     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4840
4841     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4842     return undef if !defined($product) || $product !~ s/^0x//;
4843
4844     $res = {
4845         name => $name,
4846         vendor => $vendor,
4847         product => $product,
4848         domain => $domain,
4849         bus => $bus,
4850         slot => $slot,
4851         func => $func,
4852         irq => $irq,
4853         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4854     };
4855
4856     return $res;
4857 }
4858
4859 sub pci_dev_reset {
4860     my ($dev) = @_;
4861
4862     my $name = $dev->{name};
4863
4864     my $fn = "$pcisysfs/devices/$name/reset";
4865
4866     return file_write($fn, "1");
4867 }
4868
4869 sub pci_dev_bind_to_vfio {
4870     my ($dev) = @_;
4871
4872     my $name = $dev->{name};
4873
4874     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4875
4876     if (!-d $vfio_basedir) {
4877         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4878     }
4879     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4880
4881     my $testdir = "$vfio_basedir/$name";
4882     return 1 if -d $testdir;
4883
4884     my $data = "$dev->{vendor} $dev->{product}";
4885     return undef if !file_write("$vfio_basedir/new_id", $data);
4886
4887     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4888     if (!file_write($fn, $name)) {
4889         return undef if -f $fn;
4890     }
4891
4892     $fn = "$vfio_basedir/bind";
4893     if (! -d $testdir) {
4894         return undef if !file_write($fn, $name);
4895     }
4896
4897     return -d $testdir;
4898 }
4899
4900 sub pci_dev_group_bind_to_vfio {
4901     my ($pciid) = @_;
4902
4903     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4904
4905     if (!-d $vfio_basedir) {
4906         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4907     }
4908     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4909
4910     # get IOMMU group devices
4911     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4912       my @devs = grep /^0000:/, readdir($D);
4913     closedir($D);
4914
4915     foreach my $pciid (@devs) {
4916         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4917
4918         # pci bridges, switches or root ports are not supported
4919         # they have a pci_bus subdirectory so skip them
4920         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4921
4922         my $info = pci_device_info($1);
4923         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4924     }
4925
4926     return 1;
4927 }
4928
4929 sub print_pci_addr {
4930     my ($id, $bridges) = @_;
4931
4932     my $res = '';
4933     my $devices = {
4934         piix3 => { bus => 0, addr => 1 },
4935         #addr2 : first videocard
4936         balloon0 => { bus => 0, addr => 3 },
4937         watchdog => { bus => 0, addr => 4 },
4938         scsihw0 => { bus => 0, addr => 5 },
4939         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4940         scsihw1 => { bus => 0, addr => 6 },
4941         ahci0 => { bus => 0, addr => 7 },
4942         qga0 => { bus => 0, addr => 8 },
4943         spice => { bus => 0, addr => 9 },
4944         virtio0 => { bus => 0, addr => 10 },
4945         virtio1 => { bus => 0, addr => 11 },
4946         virtio2 => { bus => 0, addr => 12 },
4947         virtio3 => { bus => 0, addr => 13 },
4948         virtio4 => { bus => 0, addr => 14 },
4949         virtio5 => { bus => 0, addr => 15 },
4950         hostpci0 => { bus => 0, addr => 16 },
4951         hostpci1 => { bus => 0, addr => 17 },
4952         net0 => { bus => 0, addr => 18 },
4953         net1 => { bus => 0, addr => 19 },
4954         net2 => { bus => 0, addr => 20 },
4955         net3 => { bus => 0, addr => 21 },
4956         net4 => { bus => 0, addr => 22 },
4957         net5 => { bus => 0, addr => 23 },
4958         vga1 => { bus => 0, addr => 24 },
4959         vga2 => { bus => 0, addr => 25 },
4960         vga3 => { bus => 0, addr => 26 },
4961         hostpci2 => { bus => 0, addr => 27 },
4962         hostpci3 => { bus => 0, addr => 28 },
4963         #addr29 : usb-host (pve-usb.cfg)
4964         'pci.1' => { bus => 0, addr => 30 },
4965         'pci.2' => { bus => 0, addr => 31 },
4966         'net6' => { bus => 1, addr => 1 },
4967         'net7' => { bus => 1, addr => 2 },
4968         'net8' => { bus => 1, addr => 3 },
4969         'net9' => { bus => 1, addr => 4 },
4970         'net10' => { bus => 1, addr => 5 },
4971         'net11' => { bus => 1, addr => 6 },
4972         'net12' => { bus => 1, addr => 7 },
4973         'net13' => { bus => 1, addr => 8 },
4974         'net14' => { bus => 1, addr => 9 },
4975         'net15' => { bus => 1, addr => 10 },
4976         'net16' => { bus => 1, addr => 11 },
4977         'net17' => { bus => 1, addr => 12 },
4978         'net18' => { bus => 1, addr => 13 },
4979         'net19' => { bus => 1, addr => 14 },
4980         'net20' => { bus => 1, addr => 15 },
4981         'net21' => { bus => 1, addr => 16 },
4982         'net22' => { bus => 1, addr => 17 },
4983         'net23' => { bus => 1, addr => 18 },
4984         'net24' => { bus => 1, addr => 19 },
4985         'net25' => { bus => 1, addr => 20 },
4986         'net26' => { bus => 1, addr => 21 },
4987         'net27' => { bus => 1, addr => 22 },
4988         'net28' => { bus => 1, addr => 23 },
4989         'net29' => { bus => 1, addr => 24 },
4990         'net30' => { bus => 1, addr => 25 },
4991         'net31' => { bus => 1, addr => 26 },
4992         'xhci' => { bus => 1, addr => 27 },
4993         'virtio6' => { bus => 2, addr => 1 },
4994         'virtio7' => { bus => 2, addr => 2 },
4995         'virtio8' => { bus => 2, addr => 3 },
4996         'virtio9' => { bus => 2, addr => 4 },
4997         'virtio10' => { bus => 2, addr => 5 },
4998         'virtio11' => { bus => 2, addr => 6 },
4999         'virtio12' => { bus => 2, addr => 7 },
5000         'virtio13' => { bus => 2, addr => 8 },
5001         'virtio14' => { bus => 2, addr => 9 },
5002         'virtio15' => { bus => 2, addr => 10 },
5003         'virtioscsi0' => { bus => 3, addr => 1 },
5004         'virtioscsi1' => { bus => 3, addr => 2 },
5005         'virtioscsi2' => { bus => 3, addr => 3 },
5006         'virtioscsi3' => { bus => 3, addr => 4 },
5007         'virtioscsi4' => { bus => 3, addr => 5 },
5008         'virtioscsi5' => { bus => 3, addr => 6 },
5009         'virtioscsi6' => { bus => 3, addr => 7 },
5010         'virtioscsi7' => { bus => 3, addr => 8 },
5011         'virtioscsi8' => { bus => 3, addr => 9 },
5012         'virtioscsi9' => { bus => 3, addr => 10 },
5013         'virtioscsi10' => { bus => 3, addr => 11 },
5014         'virtioscsi11' => { bus => 3, addr => 12 },
5015         'virtioscsi12' => { bus => 3, addr => 13 },
5016         'virtioscsi13' => { bus => 3, addr => 14 },
5017         'virtioscsi14' => { bus => 3, addr => 15 },
5018         'virtioscsi15' => { bus => 3, addr => 16 },
5019         'virtioscsi16' => { bus => 3, addr => 17 },
5020         'virtioscsi17' => { bus => 3, addr => 18 },
5021         'virtioscsi18' => { bus => 3, addr => 19 },
5022         'virtioscsi19' => { bus => 3, addr => 20 },
5023         'virtioscsi20' => { bus => 3, addr => 21 },
5024         'virtioscsi21' => { bus => 3, addr => 22 },
5025         'virtioscsi22' => { bus => 3, addr => 23 },
5026         'virtioscsi23' => { bus => 3, addr => 24 },
5027         'virtioscsi24' => { bus => 3, addr => 25 },
5028         'virtioscsi25' => { bus => 3, addr => 26 },
5029         'virtioscsi26' => { bus => 3, addr => 27 },
5030         'virtioscsi27' => { bus => 3, addr => 28 },
5031         'virtioscsi28' => { bus => 3, addr => 29 },
5032         'virtioscsi29' => { bus => 3, addr => 30 },
5033         'virtioscsi30' => { bus => 3, addr => 31 },
5034
5035     };
5036
5037     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5038            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5039            my $bus = $devices->{$id}->{bus};
5040            $res = ",bus=pci.$bus,addr=$addr";
5041            $bridges->{$bus} = 1 if $bridges;
5042     }
5043     return $res;
5044
5045 }
5046
5047 sub print_pcie_addr {
5048     my ($id) = @_;
5049
5050     my $res = '';
5051     my $devices = {
5052         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5053         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5054         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5055         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5056     };
5057
5058     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5059            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5060            my $bus = $devices->{$id}->{bus};
5061            $res = ",bus=$bus,addr=$addr";
5062     }
5063     return $res;
5064
5065 }
5066
5067 # vzdump restore implementaion
5068
5069 sub tar_archive_read_firstfile {
5070     my $archive = shift;
5071
5072     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5073
5074     # try to detect archive type first
5075     my $pid = open (TMP, "tar tf '$archive'|") ||
5076         die "unable to open file '$archive'\n";
5077     my $firstfile = <TMP>;
5078     kill 15, $pid;
5079     close TMP;
5080
5081     die "ERROR: archive contaions no data\n" if !$firstfile;
5082     chomp $firstfile;
5083
5084     return $firstfile;
5085 }
5086
5087 sub tar_restore_cleanup {
5088     my ($storecfg, $statfile) = @_;
5089
5090     print STDERR "starting cleanup\n";
5091
5092     if (my $fd = IO::File->new($statfile, "r")) {
5093         while (defined(my $line = <$fd>)) {
5094             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5095                 my $volid = $2;
5096                 eval {
5097                     if ($volid =~ m|^/|) {
5098                         unlink $volid || die 'unlink failed\n';
5099                     } else {
5100                         PVE::Storage::vdisk_free($storecfg, $volid);
5101                     }
5102                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5103                 };
5104                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5105             } else {
5106                 print STDERR "unable to parse line in statfile - $line";
5107             }
5108         }
5109         $fd->close();
5110     }
5111 }
5112
5113 sub restore_archive {
5114     my ($archive, $vmid, $user, $opts) = @_;
5115
5116     my $format = $opts->{format};
5117     my $comp;
5118
5119     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5120         $format = 'tar' if !$format;
5121         $comp = 'gzip';
5122     } elsif ($archive =~ m/\.tar$/) {
5123         $format = 'tar' if !$format;
5124     } elsif ($archive =~ m/.tar.lzo$/) {
5125         $format = 'tar' if !$format;
5126         $comp = 'lzop';
5127     } elsif ($archive =~ m/\.vma$/) {
5128         $format = 'vma' if !$format;
5129     } elsif ($archive =~ m/\.vma\.gz$/) {
5130         $format = 'vma' if !$format;
5131         $comp = 'gzip';
5132     } elsif ($archive =~ m/\.vma\.lzo$/) {
5133         $format = 'vma' if !$format;
5134         $comp = 'lzop';
5135     } else {
5136         $format = 'vma' if !$format; # default
5137     }
5138
5139     # try to detect archive format
5140     if ($format eq 'tar') {
5141         return restore_tar_archive($archive, $vmid, $user, $opts);
5142     } else {
5143         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5144     }
5145 }
5146
5147 sub restore_update_config_line {
5148     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5149
5150     return if $line =~ m/^\#qmdump\#/;
5151     return if $line =~ m/^\#vzdump\#/;
5152     return if $line =~ m/^lock:/;
5153     return if $line =~ m/^unused\d+:/;
5154     return if $line =~ m/^parent:/;
5155     return if $line =~ m/^template:/; # restored VM is never a template
5156
5157     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5158         # try to convert old 1.X settings
5159         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5160         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5161             my ($model, $macaddr) = split(/\=/, $devconfig);
5162             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5163             my $net = {
5164                 model => $model,
5165                 bridge => "vmbr$ind",
5166                 macaddr => $macaddr,
5167             };
5168             my $netstr = print_net($net);
5169
5170             print $outfd "net$cookie->{netcount}: $netstr\n";
5171             $cookie->{netcount}++;
5172         }
5173     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5174         my ($id, $netstr) = ($1, $2);
5175         my $net = parse_net($netstr);
5176         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5177         $netstr = print_net($net);
5178         print $outfd "$id: $netstr\n";
5179     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5180         my $virtdev = $1;
5181         my $value = $3;
5182         my $di = parse_drive($virtdev, $value);
5183         if (defined($di->{backup}) && !$di->{backup}) {
5184             print $outfd "#$line";
5185         } elsif ($map->{$virtdev}) {
5186             delete $di->{format}; # format can change on restore
5187             $di->{file} = $map->{$virtdev};
5188             $value = print_drive($vmid, $di);
5189             print $outfd "$virtdev: $value\n";
5190         } else {
5191             print $outfd $line;
5192         }
5193     } else {
5194         print $outfd $line;
5195     }
5196 }
5197
5198 sub scan_volids {
5199     my ($cfg, $vmid) = @_;
5200
5201     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5202
5203     my $volid_hash = {};
5204     foreach my $storeid (keys %$info) {
5205         foreach my $item (@{$info->{$storeid}}) {
5206             next if !($item->{volid} && $item->{size});
5207             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5208             $volid_hash->{$item->{volid}} = $item;
5209         }
5210     }
5211
5212     return $volid_hash;
5213 }
5214
5215 sub is_volume_in_use {
5216     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5217
5218     my $path = PVE::Storage::path($storecfg, $volid);
5219
5220     my $scan_config = sub {
5221         my ($cref, $snapname) = @_;
5222
5223         foreach my $key (keys %$cref) {
5224             my $value = $cref->{$key};
5225             if (is_valid_drivename($key)) {
5226                 next if $skip_drive && $key eq $skip_drive;
5227                 my $drive = parse_drive($key, $value);
5228                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5229                 return 1 if $volid eq $drive->{file};
5230                 if ($drive->{file} =~ m!^/!) {
5231                     return 1 if $drive->{file} eq $path;
5232                 } else {
5233                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5234                     next if !$storeid;
5235                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5236                     next if !$scfg;
5237                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5238                 }
5239             }
5240         }
5241
5242         return 0;
5243     };
5244
5245     return 1 if &$scan_config($conf);
5246
5247     undef $skip_drive;
5248
5249     foreach my $snapname (keys %{$conf->{snapshots}}) {
5250         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5251     }
5252
5253     return 0;
5254 }
5255
5256 sub update_disksize {
5257     my ($vmid, $conf, $volid_hash) = @_;
5258
5259     my $changes;
5260
5261     my $used = {};
5262
5263     # Note: it is allowed to define multiple storages with same path (alias), so
5264     # we need to check both 'volid' and real 'path' (two different volid can point
5265     # to the same path).
5266
5267     my $usedpath = {};
5268
5269     # update size info
5270     foreach my $opt (keys %$conf) {
5271         if (is_valid_drivename($opt)) {
5272             my $drive = parse_drive($opt, $conf->{$opt});
5273             my $volid = $drive->{file};
5274             next if !$volid;
5275
5276             $used->{$volid} = 1;
5277             if ($volid_hash->{$volid} &&
5278                 (my $path = $volid_hash->{$volid}->{path})) {
5279                 $usedpath->{$path} = 1;
5280             }
5281
5282             next if drive_is_cdrom($drive);
5283             next if !$volid_hash->{$volid};
5284
5285             $drive->{size} = $volid_hash->{$volid}->{size};
5286             my $new = print_drive($vmid, $drive);
5287             if ($new ne $conf->{$opt}) {
5288                 $changes = 1;
5289                 $conf->{$opt} = $new;
5290             }
5291         }
5292     }
5293
5294     # remove 'unusedX' entry if volume is used
5295     foreach my $opt (keys %$conf) {
5296         next if $opt !~ m/^unused\d+$/;
5297         my $volid = $conf->{$opt};
5298         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5299         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5300             $changes = 1;
5301             delete $conf->{$opt};
5302         }
5303     }
5304
5305     foreach my $volid (sort keys %$volid_hash) {
5306         next if $volid =~ m/vm-$vmid-state-/;
5307         next if $used->{$volid};
5308         my $path = $volid_hash->{$volid}->{path};
5309         next if !$path; # just to be sure
5310         next if $usedpath->{$path};
5311         $changes = 1;
5312         PVE::QemuConfig->add_unused_volume($conf, $volid);
5313         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5314     }
5315
5316     return $changes;
5317 }
5318
5319 sub rescan {
5320     my ($vmid, $nolock) = @_;
5321
5322     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5323
5324     my $volid_hash = scan_volids($cfg, $vmid);
5325
5326     my $updatefn =  sub {
5327         my ($vmid) = @_;
5328
5329         my $conf = PVE::QemuConfig->load_config($vmid);
5330
5331         PVE::QemuConfig->check_lock($conf);
5332
5333         my $vm_volids = {};
5334         foreach my $volid (keys %$volid_hash) {
5335             my $info = $volid_hash->{$volid};
5336             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5337         }
5338
5339         my $changes = update_disksize($vmid, $conf, $vm_volids);
5340
5341         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5342     };
5343
5344     if (defined($vmid)) {
5345         if ($nolock) {
5346             &$updatefn($vmid);
5347         } else {
5348             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5349         }
5350     } else {
5351         my $vmlist = config_list();
5352         foreach my $vmid (keys %$vmlist) {
5353             if ($nolock) {
5354                 &$updatefn($vmid);
5355             } else {
5356                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5357             }
5358         }
5359     }
5360 }
5361
5362 sub restore_vma_archive {
5363     my ($archive, $vmid, $user, $opts, $comp) = @_;
5364
5365     my $input = $archive eq '-' ? "<&STDIN" : undef;
5366     my $readfrom = $archive;
5367
5368     my $uncomp = '';
5369     if ($comp) {
5370         $readfrom = '-';
5371         my $qarchive = PVE::Tools::shellquote($archive);
5372         if ($comp eq 'gzip') {
5373             $uncomp = "zcat $qarchive|";
5374         } elsif ($comp eq 'lzop') {
5375             $uncomp = "lzop -d -c $qarchive|";
5376         } else {
5377             die "unknown compression method '$comp'\n";
5378         }
5379
5380     }
5381
5382     my $tmpdir = "/var/tmp/vzdumptmp$$";
5383     rmtree $tmpdir;
5384
5385     # disable interrupts (always do cleanups)
5386     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5387         warn "got interrupt - ignored\n";
5388     };
5389
5390     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5391     POSIX::mkfifo($mapfifo, 0600);
5392     my $fifofh;
5393
5394     my $openfifo = sub {
5395         open($fifofh, '>', $mapfifo) || die $!;
5396     };
5397
5398     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5399
5400     my $oldtimeout;
5401     my $timeout = 5;
5402
5403     my $devinfo = {};
5404
5405     my $rpcenv = PVE::RPCEnvironment::get();
5406
5407     my $conffile = PVE::QemuConfig->config_file($vmid);
5408     my $tmpfn = "$conffile.$$.tmp";
5409
5410     # Note: $oldconf is undef if VM does not exists
5411     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5412     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5413
5414     my $print_devmap = sub {
5415         my $virtdev_hash = {};
5416
5417         my $cfgfn = "$tmpdir/qemu-server.conf";
5418
5419         # we can read the config - that is already extracted
5420         my $fh = IO::File->new($cfgfn, "r") ||
5421             "unable to read qemu-server.conf - $!\n";
5422
5423         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5424         if (-f $fwcfgfn) {
5425             my $pve_firewall_dir = '/etc/pve/firewall';
5426             mkdir $pve_firewall_dir; # make sure the dir exists
5427             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5428         }
5429
5430         while (defined(my $line = <$fh>)) {
5431             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5432                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5433                 die "archive does not contain data for drive '$virtdev'\n"
5434                     if !$devinfo->{$devname};
5435                 if (defined($opts->{storage})) {
5436                     $storeid = $opts->{storage} || 'local';
5437                 } elsif (!$storeid) {
5438                     $storeid = 'local';
5439                 }
5440                 $format = 'raw' if !$format;
5441                 $devinfo->{$devname}->{devname} = $devname;
5442                 $devinfo->{$devname}->{virtdev} = $virtdev;
5443                 $devinfo->{$devname}->{format} = $format;
5444                 $devinfo->{$devname}->{storeid} = $storeid;
5445
5446                 # check permission on storage
5447                 my $pool = $opts->{pool}; # todo: do we need that?
5448                 if ($user ne 'root@pam') {
5449                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5450                 }
5451
5452                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5453             }
5454         }
5455
5456         foreach my $devname (keys %$devinfo) {
5457             die "found no device mapping information for device '$devname'\n"
5458                 if !$devinfo->{$devname}->{virtdev};
5459         }
5460
5461         my $cfg = cfs_read_file('storage.cfg');
5462
5463         # create empty/temp config
5464         if ($oldconf) {
5465             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5466             foreach_drive($oldconf, sub {
5467                 my ($ds, $drive) = @_;
5468
5469                 return if drive_is_cdrom($drive);
5470
5471                 my $volid = $drive->{file};
5472
5473                 return if !$volid || $volid =~ m|^/|;
5474
5475                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5476                 return if !$path || !$owner || ($owner != $vmid);
5477
5478                 # Note: only delete disk we want to restore
5479                 # other volumes will become unused
5480                 if ($virtdev_hash->{$ds}) {
5481                     PVE::Storage::vdisk_free($cfg, $volid);
5482                 }
5483             });
5484
5485             # delete vmstate files
5486             # since after the restore we have no snapshots anymore
5487             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5488                 my $snap = $oldconf->{snapshots}->{$snapname};
5489                 if ($snap->{vmstate}) {
5490                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5491                     if (my $err = $@) {
5492                         warn $err;
5493                     }
5494                 }
5495             }
5496         }
5497
5498         my $map = {};
5499         foreach my $virtdev (sort keys %$virtdev_hash) {
5500             my $d = $virtdev_hash->{$virtdev};
5501             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5502             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5503
5504             # test if requested format is supported
5505             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5506             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5507             $d->{format} = $defFormat if !$supported;
5508
5509             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5510                                                   $d->{format}, undef, $alloc_size);
5511             print STDERR "new volume ID is '$volid'\n";
5512             $d->{volid} = $volid;
5513             my $path = PVE::Storage::path($cfg, $volid);
5514
5515             PVE::Storage::activate_volumes($cfg,[$volid]);
5516
5517             my $write_zeros = 1;
5518             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5519                 $write_zeros = 0;
5520             }
5521
5522             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5523
5524             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5525             $map->{$virtdev} = $volid;
5526         }
5527
5528         $fh->seek(0, 0) || die "seek failed - $!\n";
5529
5530         my $outfd = new IO::File ($tmpfn, "w") ||
5531             die "unable to write config for VM $vmid\n";
5532
5533         my $cookie = { netcount => 0 };
5534         while (defined(my $line = <$fh>)) {
5535             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5536         }
5537
5538         $fh->close();
5539         $outfd->close();
5540     };
5541
5542     eval {
5543         # enable interrupts
5544         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5545             die "interrupted by signal\n";
5546         };
5547         local $SIG{ALRM} = sub { die "got timeout\n"; };
5548
5549         $oldtimeout = alarm($timeout);
5550
5551         my $parser = sub {
5552             my $line = shift;
5553
5554             print "$line\n";
5555
5556             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5557                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5558                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5559             } elsif ($line =~ m/^CTIME: /) {
5560                 # we correctly received the vma config, so we can disable
5561                 # the timeout now for disk allocation (set to 10 minutes, so
5562                 # that we always timeout if something goes wrong)
5563                 alarm(600);
5564                 &$print_devmap();
5565                 print $fifofh "done\n";
5566                 my $tmp = $oldtimeout || 0;
5567                 $oldtimeout = undef;
5568                 alarm($tmp);
5569                 close($fifofh);
5570             }
5571         };
5572
5573         print "restore vma archive: $cmd\n";
5574         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5575     };
5576     my $err = $@;
5577
5578     alarm($oldtimeout) if $oldtimeout;
5579
5580     my $vollist = [];
5581     foreach my $devname (keys %$devinfo) {
5582         my $volid = $devinfo->{$devname}->{volid};
5583         push @$vollist, $volid if $volid;
5584     }
5585
5586     my $cfg = cfs_read_file('storage.cfg');
5587     PVE::Storage::deactivate_volumes($cfg, $vollist);
5588
5589     unlink $mapfifo;
5590
5591     if ($err) {
5592         rmtree $tmpdir;
5593         unlink $tmpfn;
5594
5595         foreach my $devname (keys %$devinfo) {
5596             my $volid = $devinfo->{$devname}->{volid};
5597             next if !$volid;
5598             eval {
5599                 if ($volid =~ m|^/|) {
5600                     unlink $volid || die 'unlink failed\n';
5601                 } else {
5602                     PVE::Storage::vdisk_free($cfg, $volid);
5603                 }
5604                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5605             };
5606             print STDERR "unable to cleanup '$volid' - $@" if $@;
5607         }
5608         die $err;
5609     }
5610
5611     rmtree $tmpdir;
5612
5613     rename($tmpfn, $conffile) ||
5614         die "unable to commit configuration file '$conffile'\n";
5615
5616     PVE::Cluster::cfs_update(); # make sure we read new file
5617
5618     eval { rescan($vmid, 1); };
5619     warn $@ if $@;
5620 }
5621
5622 sub restore_tar_archive {
5623     my ($archive, $vmid, $user, $opts) = @_;
5624
5625     if ($archive ne '-') {
5626         my $firstfile = tar_archive_read_firstfile($archive);
5627         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5628             if $firstfile ne 'qemu-server.conf';
5629     }
5630
5631     my $storecfg = cfs_read_file('storage.cfg');
5632
5633     # destroy existing data - keep empty config
5634     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5635     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5636
5637     my $tocmd = "/usr/lib/qemu-server/qmextract";
5638
5639     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5640     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5641     $tocmd .= ' --prealloc' if $opts->{prealloc};
5642     $tocmd .= ' --info' if $opts->{info};
5643
5644     # tar option "xf" does not autodetect compression when read from STDIN,
5645     # so we pipe to zcat
5646     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5647         PVE::Tools::shellquote("--to-command=$tocmd");
5648
5649     my $tmpdir = "/var/tmp/vzdumptmp$$";
5650     mkpath $tmpdir;
5651
5652     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5653     local $ENV{VZDUMP_VMID} = $vmid;
5654     local $ENV{VZDUMP_USER} = $user;
5655
5656     my $conffile = PVE::QemuConfig->config_file($vmid);
5657     my $tmpfn = "$conffile.$$.tmp";
5658
5659     # disable interrupts (always do cleanups)
5660     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5661         print STDERR "got interrupt - ignored\n";
5662     };
5663
5664     eval {
5665         # enable interrupts
5666         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5667             die "interrupted by signal\n";
5668         };
5669
5670         if ($archive eq '-') {
5671             print "extracting archive from STDIN\n";
5672             run_command($cmd, input => "<&STDIN");
5673         } else {
5674             print "extracting archive '$archive'\n";
5675             run_command($cmd);
5676         }
5677
5678         return if $opts->{info};
5679
5680         # read new mapping
5681         my $map = {};
5682         my $statfile = "$tmpdir/qmrestore.stat";
5683         if (my $fd = IO::File->new($statfile, "r")) {
5684             while (defined (my $line = <$fd>)) {
5685                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5686                     $map->{$1} = $2 if $1;
5687                 } else {
5688                     print STDERR "unable to parse line in statfile - $line\n";
5689                 }
5690             }
5691             $fd->close();
5692         }
5693
5694         my $confsrc = "$tmpdir/qemu-server.conf";
5695
5696         my $srcfd = new IO::File($confsrc, "r") ||
5697             die "unable to open file '$confsrc'\n";
5698
5699         my $outfd = new IO::File ($tmpfn, "w") ||
5700             die "unable to write config for VM $vmid\n";
5701
5702         my $cookie = { netcount => 0 };
5703         while (defined (my $line = <$srcfd>)) {
5704             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5705         }
5706
5707         $srcfd->close();
5708         $outfd->close();
5709     };
5710     my $err = $@;
5711
5712     if ($err) {
5713
5714         unlink $tmpfn;
5715
5716         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5717
5718         die $err;
5719     }
5720
5721     rmtree $tmpdir;
5722
5723     rename $tmpfn, $conffile ||
5724         die "unable to commit configuration file '$conffile'\n";
5725
5726     PVE::Cluster::cfs_update(); # make sure we read new file
5727
5728     eval { rescan($vmid, 1); };
5729     warn $@ if $@;
5730 };
5731
5732 sub foreach_writable_storage {
5733     my ($conf, $func) = @_;
5734
5735     my $sidhash = {};
5736
5737     foreach my $ds (keys %$conf) {
5738         next if !is_valid_drivename($ds);
5739
5740         my $drive = parse_drive($ds, $conf->{$ds});
5741         next if !$drive;
5742         next if drive_is_cdrom($drive);
5743
5744         my $volid = $drive->{file};
5745
5746         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5747         $sidhash->{$sid} = $sid if $sid;
5748     }
5749
5750     foreach my $sid (sort keys %$sidhash) {
5751         &$func($sid);
5752     }
5753 }
5754
5755 sub do_snapshots_with_qemu {
5756     my ($storecfg, $volid) = @_;
5757
5758     my $storage_name = PVE::Storage::parse_volume_id($volid);
5759
5760     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5761         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5762         return 1;
5763     }
5764
5765     if ($volid =~ m/\.(qcow2|qed)$/){
5766         return 1;
5767     }
5768
5769     return undef;
5770 }
5771
5772 sub qga_check_running {
5773     my ($vmid) = @_;
5774
5775     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5776     if ($@) {
5777         warn "Qemu Guest Agent are not running - $@";
5778         return 0;
5779     }
5780     return 1;
5781 }
5782
5783 sub template_create {
5784     my ($vmid, $conf, $disk) = @_;
5785
5786     my $storecfg = PVE::Storage::config();
5787
5788     foreach_drive($conf, sub {
5789         my ($ds, $drive) = @_;
5790
5791         return if drive_is_cdrom($drive);
5792         return if $disk && $ds ne $disk;
5793
5794         my $volid = $drive->{file};
5795         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5796
5797         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5798         $drive->{file} = $voliddst;
5799         $conf->{$ds} = print_drive($vmid, $drive);
5800         PVE::QemuConfig->write_config($vmid, $conf);
5801     });
5802 }
5803
5804 sub qemu_img_convert {
5805     my ($src_volid, $dst_volid, $size, $snapname) = @_;
5806
5807     my $storecfg = PVE::Storage::config();
5808     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5809     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5810
5811     if ($src_storeid && $dst_storeid) {
5812
5813         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5814
5815         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5816         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5817
5818         my $src_format = qemu_img_format($src_scfg, $src_volname);
5819         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5820
5821         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5822         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5823
5824         my $cmd = [];
5825         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5826         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5827         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
5828
5829         my $parser = sub {
5830             my $line = shift;
5831             if($line =~ m/\((\S+)\/100\%\)/){
5832                 my $percent = $1;
5833                 my $transferred = int($size * $percent / 100);
5834                 my $remaining = $size - $transferred;
5835
5836                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5837             }
5838
5839         };
5840
5841         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5842         my $err = $@;
5843         die "copy failed: $err" if $err;
5844     }
5845 }
5846
5847 sub qemu_img_format {
5848     my ($scfg, $volname) = @_;
5849
5850     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5851         return $1;
5852     } else {
5853         return "raw";
5854     }
5855 }
5856
5857 sub qemu_drive_mirror {
5858     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
5859
5860     my $storecfg = PVE::Storage::config();
5861     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5862
5863     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5864
5865     my $format = qemu_img_format($dst_scfg, $dst_volname);
5866
5867     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5868
5869     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
5870     $opts->{format} = $format if $format;
5871
5872     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5873
5874     eval {
5875     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5876         while (1) {
5877             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5878             my $stat = @$stats[0];
5879             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5880             die "error job is not mirroring" if $stat->{type} ne "mirror";
5881
5882             my $busy = $stat->{busy};
5883             my $ready = $stat->{ready};
5884
5885             if (my $total = $stat->{len}) {
5886                 my $transferred = $stat->{offset} || 0;
5887                 my $remaining = $total - $transferred;
5888                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5889
5890                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5891             }
5892
5893
5894             if ($stat->{ready} eq 'true') {
5895
5896                 last if $vmiddst != $vmid;
5897
5898                 # try to switch the disk if source and destination are on the same guest
5899                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5900                 last if !$@;
5901                 die $@ if $@ !~ m/cannot be completed/;
5902             }
5903             sleep 1;
5904         }
5905
5906
5907     };
5908     my $err = $@;
5909
5910     my $cancel_job = sub {
5911         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5912         while (1) {
5913             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5914             my $stat = @$stats[0];
5915             last if !$stat;
5916             sleep 1;
5917         }
5918     };
5919
5920     if ($err) {
5921         eval { &$cancel_job(); };
5922         die "mirroring error: $err";
5923     }
5924
5925     if ($vmiddst != $vmid) {
5926         # if we clone a disk for a new target vm, we don't switch the disk
5927         &$cancel_job(); # so we call block-job-cancel
5928     }
5929 }
5930
5931 sub clone_disk {
5932     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5933         $newvmid, $storage, $format, $full, $newvollist) = @_;
5934
5935     my $newvolid;
5936
5937     if (!$full) {
5938         print "create linked clone of drive $drivename ($drive->{file})\n";
5939         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5940         push @$newvollist, $newvolid;
5941     } else {
5942         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5943         $storeid = $storage if $storage;
5944
5945         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5946         if (!$format) {
5947             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5948             $format = qemu_img_format($scfg, $volname);
5949         }
5950
5951         # test if requested format is supported - else use default
5952         my $supported = grep { $_ eq $format } @$validFormats;
5953         $format = $defFormat if !$supported;
5954
5955         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5956
5957         print "create full clone of drive $drivename ($drive->{file})\n";
5958         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5959         push @$newvollist, $newvolid;
5960
5961         PVE::Storage::activate_volumes($storecfg, $newvollist);
5962
5963         if (!$running || $snapname) {
5964             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
5965         } else {
5966             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
5967         }
5968     }
5969
5970     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5971
5972     my $disk = $drive;
5973     $disk->{format} = undef;
5974     $disk->{file} = $newvolid;
5975     $disk->{size} = $size;
5976
5977     return $disk;
5978 }
5979
5980 # this only works if VM is running
5981 sub get_current_qemu_machine {
5982     my ($vmid) = @_;
5983
5984     my $cmd = { execute => 'query-machines', arguments => {} };
5985     my $res = vm_qmp_command($vmid, $cmd);
5986
5987     my ($current, $default);
5988     foreach my $e (@$res) {
5989         $default = $e->{name} if $e->{'is-default'};
5990         $current = $e->{name} if $e->{'is-current'};
5991     }
5992
5993     # fallback to the default machine if current is not supported by qemu
5994     return $current || $default || 'pc';
5995 }
5996
5997 sub qemu_machine_feature_enabled {
5998     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5999
6000     my $current_major;
6001     my $current_minor;
6002
6003     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6004
6005         $current_major = $3;
6006         $current_minor = $4;
6007
6008     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6009
6010         $current_major = $1;
6011         $current_minor = $2;
6012     }
6013
6014     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6015
6016
6017 }
6018
6019 sub qemu_machine_pxe {
6020     my ($vmid, $conf, $machine) = @_;
6021
6022     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6023
6024     foreach my $opt (keys %$conf) {
6025         next if $opt !~ m/^net(\d+)$/;
6026         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6027         next if !$net;
6028         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6029         return $machine.".pxe" if $romfile =~ m/pxe/;
6030         last;
6031     }
6032
6033     return $machine;
6034 }
6035
6036 sub qemu_use_old_bios_files {
6037     my ($machine_type) = @_;
6038
6039     return if !$machine_type;
6040
6041     my $use_old_bios_files = undef;
6042
6043     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6044         $machine_type = $1;
6045         $use_old_bios_files = 1;
6046     } else {
6047         my $kvmver = kvm_user_version();
6048         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6049         # load new efi bios files on migration. So this hack is required to allow
6050         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6051         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6052         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6053     }
6054
6055     return ($use_old_bios_files, $machine_type);
6056 }
6057
6058 sub lspci {
6059
6060     my $devices = {};
6061
6062     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6063             my (undef, $id, $function) = @_;
6064             my $res = { id => $id, function => $function};
6065             push @{$devices->{$id}}, $res;
6066     });
6067
6068     return $devices;
6069 }
6070
6071 sub vm_iothreads_list {
6072     my ($vmid) = @_;
6073
6074     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6075
6076     my $iothreads = {};
6077     foreach my $iothread (@$res) {
6078         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6079     }
6080
6081     return $iothreads;
6082 }
6083
6084 sub scsihw_infos {
6085     my ($conf, $drive) = @_;
6086
6087     my $maxdev = 0;
6088
6089     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6090         $maxdev = 7;
6091     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6092         $maxdev = 1;
6093     } else {
6094         $maxdev = 256;
6095     }
6096
6097     my $controller = int($drive->{index} / $maxdev);
6098     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6099
6100     return ($maxdev, $controller, $controller_prefix);
6101 }
6102
6103 # bash completion helper
6104
6105 sub complete_backup_archives {
6106     my ($cmdname, $pname, $cvalue) = @_;
6107
6108     my $cfg = PVE::Storage::config();
6109
6110     my $storeid;
6111
6112     if ($cvalue =~ m/^([^:]+):/) {
6113         $storeid = $1;
6114     }
6115
6116     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6117
6118     my $res = [];
6119     foreach my $id (keys %$data) {
6120         foreach my $item (@{$data->{$id}}) {
6121             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6122             push @$res, $item->{volid} if defined($item->{volid});
6123         }
6124     }
6125
6126     return $res;
6127 }
6128
6129 my $complete_vmid_full = sub {
6130     my ($running) = @_;
6131
6132     my $idlist = vmstatus();
6133
6134     my $res = [];
6135
6136     foreach my $id (keys %$idlist) {
6137         my $d = $idlist->{$id};
6138         if (defined($running)) {
6139             next if $d->{template};
6140             next if $running && $d->{status} ne 'running';
6141             next if !$running && $d->{status} eq 'running';
6142         }
6143         push @$res, $id;
6144
6145     }
6146     return $res;
6147 };
6148
6149 sub complete_vmid {
6150     return &$complete_vmid_full();
6151 }
6152
6153 sub complete_vmid_stopped {
6154     return &$complete_vmid_full(0);
6155 }
6156
6157 sub complete_vmid_running {
6158     return &$complete_vmid_full(1);
6159 }
6160
6161 sub complete_storage {
6162
6163     my $cfg = PVE::Storage::config();
6164     my $ids = $cfg->{ids};
6165
6166     my $res = [];
6167     foreach my $sid (keys %$ids) {
6168         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6169         next if !$ids->{$sid}->{content}->{images};
6170         push @$res, $sid;
6171     }
6172
6173     return $res;
6174 }
6175
6176 1;