PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35 use URI::Escape;
  36
  37 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  38
  39 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  40
  41 # Note about locking: we use flock on the config file protect
  42 # against concurent actions.
  43 # Aditionaly, we have a 'lock' setting in the config file. This
  44 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  45 # allowed when such lock is set. But you can ignore this kind of
  46 # lock with the --skiplock flag.
  47
  48 cfs_register_file('/qemu-server/',
  49                   \&parse_vm_config,
  50                   \&write_vm_config);
  51
  52 PVE::JSONSchema::register_standard_option('skiplock', {
  53     description => "Ignore locks - only root is allowed to use this option.",
  54     type => 'boolean',
  55     optional => 1,
  56 });
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 #no warnings 'redefine';
  72
  73 sub cgroups_write {
  74    my ($controller, $vmid, $option, $value) = @_;
  75
  76    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  77    PVE::ProcFSTools::write_proc_entry($path, $value);
  78
  79 }
  80
  81 my $nodename = PVE::INotify::nodename();
  82
  83 mkdir "/etc/pve/nodes/$nodename";
  84 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  85 mkdir $confdir;
  86
  87 my $var_run_tmpdir = "/var/run/qemu-server";
  88 mkdir $var_run_tmpdir;
  89
  90 my $lock_dir = "/var/lock/qemu-server";
  91 mkdir $lock_dir;
  92
  93 my $pcisysfs = "/sys/bus/pci";
  94
  95 my $cpu_fmt = {
  96     cputype => {
  97         description => "Emulated CPU type.",
  98         type => 'string',
  99         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 100         default => 'kvm64',
 101         default_key => 1,
 102     },
 103     hidden => {
 104         description => "Do not identify as a KVM virtual machine.",
 105         type => 'boolean',
 106         optional => 1,
 107         default => 0
 108     },
 109 };
 110
 111 my $confdesc = {
 112     onboot => {
 113         optional => 1,
 114         type => 'boolean',
 115         description => "Specifies whether a VM will be started during system bootup.",
 116         default => 0,
 117     },
 118     autostart => {
 119         optional => 1,
 120         type => 'boolean',
 121         description => "Automatic restart after crash (currently ignored).",
 122         default => 0,
 123     },
 124     hotplug => {
 125         optional => 1,
 126         type => 'string', format => 'pve-hotplug-features',
 127         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 128         default => 'network,disk,usb',
 129     },
 130     reboot => {
 131         optional => 1,
 132         type => 'boolean',
 133         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 134         default => 1,
 135     },
 136     lock => {
 137         optional => 1,
 138         type => 'string',
 139         description => "Lock/unlock the VM.",
 140         enum => [qw(migrate backup snapshot rollback)],
 141     },
 142     cpulimit => {
 143         optional => 1,
 144         type => 'number',
 145         description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 146         minimum => 0,
 147         maximum => 128,
 148         default => 0,
 149     },
 150     cpuunits => {
 151         optional => 1,
 152         type => 'integer',
 153         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 154         minimum => 0,
 155         maximum => 500000,
 156         default => 1000,
 157     },
 158     memory => {
 159         optional => 1,
 160         type => 'integer',
 161         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 162         minimum => 16,
 163         default => 512,
 164     },
 165     balloon => {
 166         optional => 1,
 167         type => 'integer',
 168         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 169         minimum => 0,
 170     },
 171     shares => {
 172         optional => 1,
 173         type => 'integer',
 174         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 175         minimum => 0,
 176         maximum => 50000,
 177         default => 1000,
 178     },
 179     keyboard => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 183         enum => PVE::Tools::kvmkeymaplist(),
 184         default => 'en-us',
 185     },
 186     name => {
 187         optional => 1,
 188         type => 'string', format => 'dns-name',
 189         description => "Set a name for the VM. Only used on the configuration web interface.",
 190     },
 191     scsihw => {
 192         optional => 1,
 193         type => 'string',
 194         description => "scsi controller model",
 195         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 196         default => 'lsi',
 197     },
 198     description => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 202     },
 203     ostype => {
 204         optional => 1,
 205         type => 'string',
 206         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 207         description => <<EODESC,
 208 Used to enable special optimization/features for specific
 209 operating systems:
 210
 211 other  => unspecified OS
 212 wxp    => Microsoft Windows XP
 213 w2k    => Microsoft Windows 2000
 214 w2k3   => Microsoft Windows 2003
 215 w2k8   => Microsoft Windows 2008
 216 wvista => Microsoft Windows Vista
 217 win7   => Microsoft Windows 7
 218 win8   => Microsoft Windows 8/2012
 219 l24    => Linux 2.4 Kernel
 220 l26    => Linux 2.6/3.X Kernel
 221 solaris => solaris/opensolaris/openindiania kernel
 222
 223 other|l24|l26|solaris                       ... no special behaviour
 224 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 225 EODESC
 226     },
 227     boot => {
 228         optional => 1,
 229         type => 'string',
 230         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 231         pattern => '[acdn]{1,4}',
 232         default => 'cdn',
 233     },
 234     bootdisk => {
 235         optional => 1,
 236         type => 'string', format => 'pve-qm-bootdisk',
 237         description => "Enable booting from specified disk.",
 238         pattern => '(ide|sata|scsi|virtio)\d+',
 239     },
 240     smp => {
 241         optional => 1,
 242         type => 'integer',
 243         description => "The number of CPUs. Please use option -sockets instead.",
 244         minimum => 1,
 245         default => 1,
 246     },
 247     sockets => {
 248         optional => 1,
 249         type => 'integer',
 250         description => "The number of CPU sockets.",
 251         minimum => 1,
 252         default => 1,
 253     },
 254     cores => {
 255         optional => 1,
 256         type => 'integer',
 257         description => "The number of cores per socket.",
 258         minimum => 1,
 259         default => 1,
 260     },
 261     numa => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable NUMA.",
 265         default => 0,
 266     },
 267     vcpus => {
 268         optional => 1,
 269         type => 'integer',
 270         description => "Number of hotplugged vcpus.",
 271         minimum => 1,
 272         default => 0,
 273     },
 274     acpi => {
 275         optional => 1,
 276         type => 'boolean',
 277         description => "Enable/disable ACPI.",
 278         default => 1,
 279     },
 280     agent => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Enable/disable Qemu GuestAgent.",
 284         default => 0,
 285     },
 286     kvm => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Enable/disable KVM hardware virtualization.",
 290         default => 1,
 291     },
 292     tdf => {
 293         optional => 1,
 294         type => 'boolean',
 295         description => "Enable/disable time drift fix.",
 296         default => 0,
 297     },
 298     localtime => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 302     },
 303     freeze => {
 304         optional => 1,
 305         type => 'boolean',
 306         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 307     },
 308     vga => {
 309         optional => 1,
 310         type => 'string',
 311         description => "Select the VGA type. If you want to use high resolution" .
 312             " modes (>= 1280x1024x16) then you should use the options " .
 313             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 314             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 315             "display sever. For win* OS you can select how many independent " .
 316             "displays you want, Linux guests can add displays them self. " .
 317             "You can also run without any graphic card, using a serial device" .
 318             " as terminal.",
 319         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 320     },
 321     watchdog => {
 322         optional => 1,
 323         type => 'string', format => 'pve-qm-watchdog',
 324         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 325         description => "Create a virtual hardware watchdog device. Once enabled" .
 326             " (by a guest action), the watchdog must be periodically polled " .
 327             "by an agent inside the guest or else the watchdog will reset " .
 328             "the guest (or execute the respective action specified)",
 329     },
 330     startdate => {
 331         optional => 1,
 332         type => 'string',
 333         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 334         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 335         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 336         default => 'now',
 337     },
 338     startup =>  get_standard_option('pve-startup-order'),
 339     template => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Enable/disable Template.",
 343         default => 0,
 344     },
 345     args => {
 346         optional => 1,
 347         type => 'string',
 348         description => <<EODESCR,
 349 NOTE: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 350
 351 args: -no-reboot -no-hpet
 352 EODESCR
 353     },
 354     tablet => {
 355         optional => 1,
 356         type => 'boolean',
 357         default => 1,
 358         description => "Enable/disable the USB tablet device. This device is " .
 359             "usually needed to allow absolute mouse positioning with VNC. " .
 360             "Else the mouse runs out of sync with normal VNC clients. " .
 361             "If you're running lots of console-only guests on one host, " .
 362             "you may consider disabling this to save some context switches. " .
 363             "This is turned off by default if you use spice (-vga=qxl).",
 364     },
 365     migrate_speed => {
 366         optional => 1,
 367         type => 'integer',
 368         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 369         minimum => 0,
 370         default => 0,
 371     },
 372     migrate_downtime => {
 373         optional => 1,
 374         type => 'number',
 375         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 376         minimum => 0,
 377         default => 0.1,
 378     },
 379     cdrom => {
 380         optional => 1,
 381         type => 'string', format => 'pve-qm-drive',
 382         typetext => 'volume',
 383         description => "This is an alias for option -ide2",
 384     },
 385     cpu => {
 386         optional => 1,
 387         description => "Emulated CPU type.",
 388         type => 'string',
 389         format => $cpu_fmt,
 390     },
 391     parent => get_standard_option('pve-snapshot-name', {
 392         optional => 1,
 393         description => "Parent snapshot name. This is used internally, and should not be modified.",
 394     }),
 395     snaptime => {
 396         optional => 1,
 397         description => "Timestamp for snapshots.",
 398         type => 'integer',
 399         minimum => 0,
 400     },
 401     vmstate => {
 402         optional => 1,
 403         type => 'string', format => 'pve-volume-id',
 404         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 405     },
 406     machine => {
 407         description => "Specific the Qemu machine type.",
 408         type => 'string',
 409         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 410         maxLength => 40,
 411         optional => 1,
 412     },
 413     smbios1 => {
 414         description => "Specify SMBIOS type 1 fields.",
 415         type => 'string', format => 'pve-qm-smbios1',
 416         maxLength => 256,
 417         optional => 1,
 418     },
 419     protection => {
 420         optional => 1,
 421         type => 'boolean',
 422         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 423         default => 0,
 424     },
 425     bios => {
 426         optional => 1,
 427         type => 'string',
 428         enum => [ qw(seabios ovmf) ],
 429         description => "Select BIOS implementation.",
 430         default => 'seabios',
 431     },
 432 };
 433
 434 # what about other qemu settings ?
 435 #cpu => 'string',
 436 #machine => 'string',
 437 #fda => 'file',
 438 #fdb => 'file',
 439 #mtdblock => 'file',
 440 #sd => 'file',
 441 #pflash => 'file',
 442 #snapshot => 'bool',
 443 #bootp => 'file',
 444 ##tftp => 'dir',
 445 ##smb => 'dir',
 446 #kernel => 'file',
 447 #append => 'string',
 448 #initrd => 'file',
 449 ##soundhw => 'string',
 450
 451 while (my ($k, $v) = each %$confdesc) {
 452     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 453 }
 454
 455 my $MAX_IDE_DISKS = 4;
 456 my $MAX_SCSI_DISKS = 14;
 457 my $MAX_VIRTIO_DISKS = 16;
 458 my $MAX_SATA_DISKS = 6;
 459 my $MAX_USB_DEVICES = 5;
 460 my $MAX_NETS = 32;
 461 my $MAX_UNUSED_DISKS = 8;
 462 my $MAX_HOSTPCI_DEVICES = 4;
 463 my $MAX_SERIAL_PORTS = 4;
 464 my $MAX_PARALLEL_PORTS = 3;
 465 my $MAX_NUMA = 8;
 466 my $MAX_MEM = 4194304;
 467 my $STATICMEM = 1024;
 468
 469 my $numadesc = {
 470     optional => 1,
 471     type => 'string', format => 'pve-qm-numanode',
 472     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 473     description => "numa topology",
 474 };
 475 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 476
 477 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 478     $confdesc->{"numa$i"} = $numadesc;
 479 }
 480
 481 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 482                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 483                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 484 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 485
 486 my $netdesc = {
 487     optional => 1,
 488     type => 'string', format => 'pve-qm-net',
 489     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 490     description => <<EODESCR,
 491 Specify network devices.
 492
 493 MODEL is one of: $nic_model_list_txt
 494
 495 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 496 automatically generated if not specified.
 497
 498 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 499
 500 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 501
 502 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 503
 504 10.0.2.2   Gateway
 505 10.0.2.3   DNS Server
 506 10.0.2.4   SMB Server
 507
 508 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 509
 510 EODESCR
 511 };
 512 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 513
 514 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 515     $confdesc->{"net$i"} = $netdesc;
 516 }
 517
 518 my $drivename_hash;
 519
 520 my %drivedesc_base = (
 521     volume => { alias => 'file' },
 522     file => {
 523         type => 'string',
 524         format => 'pve-volume-id',
 525         default_key => 1,
 526         format_description => 'volume',
 527         description => "The drive's backing volume.",
 528     },
 529     media => {
 530         type => 'string',
 531         format_description => 'cdrom|disk',
 532         enum => [qw(cdrom disk)],
 533         description => "The drive's media type.",
 534         default => 'disk',
 535         optional => 1
 536     },
 537     cyls => {
 538         type => 'integer',
 539         format_description => 'count',
 540         description => "Force the drive's physical geometry to have a specific cylinder count.",
 541         optional => 1
 542     },
 543     heads => {
 544         type => 'integer',
 545         format_description => 'count',
 546         description => "Force the drive's physical geometry to have a specific head count.",
 547         optional => 1
 548     },
 549     secs => {
 550         type => 'integer',
 551         format_description => 'count',
 552         description => "Force the drive's physical geometry to have a specific sector count.",
 553         optional => 1
 554     },
 555     trans => {
 556         type => 'string',
 557         format_description => 'none|lba|auto',
 558         enum => [qw(none lba auto)],
 559         description => "Force disk geometry bios translation mode.",
 560         optional => 1,
 561     },
 562     snapshot => {
 563         type => 'boolean',
 564         format_description => 'on|off',
 565         description => "Whether the drive should be included when making snapshots.",
 566         optional => 1,
 567     },
 568     cache => {
 569         type => 'string',
 570         format_description => 'none|writethrough|writeback|unsafe|directsync',
 571         enum => [qw(none writethrough writeback unsafe directsync)],
 572         description => "The drive's cache mode",
 573         optional => 1,
 574     },
 575     format => {
 576         type => 'string',
 577         format_description => 'drive format',
 578         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 579         description => "The drive's backing file's data format.",
 580         optional => 1,
 581     },
 582     size => {
 583         type => 'string',
 584         format => 'disk-size',
 585         description => "Disk size. This is purely informational and has no effect.",
 586         optional => 1,
 587     },
 588     backup => {
 589         type => 'boolean',
 590         format_description => 'on|off',
 591         description => "Whether the drive should be included when making backups.",
 592         optional => 1,
 593     },
 594     werror => {
 595         type => 'string',
 596         format_description => 'enospc|ignore|report|stop',
 597         enum => [qw(enospc ignore report stop)],
 598         description => 'Write error action.',
 599         optional => 1,
 600     },
 601     aio => {
 602         type => 'string',
 603         format_description => 'native|threads',
 604         enum => [qw(native threads)],
 605         description => 'AIO type to use.',
 606         optional => 1,
 607     },
 608     discard => {
 609         type => 'string',
 610         format_description => 'ignore|on',
 611         enum => [qw(ignore on)],
 612         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 613         optional => 1,
 614     },
 615     detect_zeroes => {
 616         type => 'boolean',
 617         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 618         optional => 1,
 619     },
 620     serial => {
 621         type => 'string',
 622         format => 'urlencoded',
 623         format_description => 'serial',
 624         maxLength => 20*3, # *3 since it's %xx url enoded
 625         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 626         optional => 1,
 627     }
 628 );
 629
 630 my %rerror_fmt = (
 631     rerror => {
 632         type => 'string',
 633         format_description => 'ignore|report|stop',
 634         enum => [qw(ignore report stop)],
 635         description => 'Read error action.',
 636         optional => 1,
 637     },
 638 );
 639
 640 my %iothread_fmt = ( iothread => {
 641         type => 'boolean',
 642         format_description => 'off|on',
 643         description => "Whether to use iothreads for this drive",
 644         optional => 1,
 645 });
 646
 647 my %model_fmt = (
 648     model => {
 649         type => 'string',
 650         format => 'urlencoded',
 651         format_description => 'model',
 652         maxLength => 40*3, # *3 since it's %xx url enoded
 653         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 654         optional => 1,
 655     },
 656 );
 657
 658 my %queues_fmt = (
 659     queues => {
 660         type => 'integer',
 661         format_description => 'nbqueues',
 662         description => "Number of queues.",
 663         minimum => 2,
 664         optional => 1
 665     }
 666 );
 667
 668 my $add_throttle_desc = sub {
 669     my ($key, $type, $what, $size, $longsize) = @_;
 670     $drivedesc_base{$key} = {
 671         type => $type,
 672         format_description => $size,
 673         description => "Maximum $what speed in $longsize per second.",
 674         optional => 1,
 675     };
 676 };
 677 # throughput: (leaky bucket)
 678 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 679 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 680 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 681 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 682 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 683 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 684 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 685 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 686 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 687
 688 # pools: (pool of IO before throttling starts taking effect)
 689 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 690 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 691 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 692 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 693 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 694 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 695
 696 my $ide_fmt = {
 697     %drivedesc_base,
 698     %rerror_fmt,
 699     %model_fmt,
 700 };
 701
 702 my $idedesc = {
 703     optional => 1,
 704     type => 'string', format => $ide_fmt,
 705     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 706 };
 707 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 708
 709 my $scsi_fmt = {
 710     %drivedesc_base,
 711     %iothread_fmt,
 712     %queues_fmt,
 713 };
 714 my $scsidesc = {
 715     optional => 1,
 716     type => 'string', format => $scsi_fmt,
 717     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 718 };
 719 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 720
 721 my $sata_fmt = {
 722     %drivedesc_base,
 723     %rerror_fmt,
 724 };
 725 my $satadesc = {
 726     optional => 1,
 727     type => 'string', format => $sata_fmt,
 728     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 729 };
 730 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 731
 732 my $virtio_fmt = {
 733     %drivedesc_base,
 734     %iothread_fmt,
 735     %rerror_fmt,
 736 };
 737 my $virtiodesc = {
 738     optional => 1,
 739     type => 'string', format => $virtio_fmt,
 740     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 741 };
 742 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 743
 744 my $alldrive_fmt = {
 745     %drivedesc_base,
 746     %rerror_fmt,
 747     %iothread_fmt,
 748     %model_fmt,
 749     %queues_fmt,
 750 };
 751
 752 my $usb_fmt = {
 753     host => {
 754         default_key => 1,
 755         type => 'string', format => 'pve-qm-usb-device',
 756         format_description => 'HOSTUSBDEVICE|spice',
 757         description => 'The Host USB device or port or the value spice',
 758     },
 759     usb3 => {
 760         optional => 1,
 761         type => 'boolean',
 762         format_description => 'yes|no',
 763         description => 'Specifies whether if given host option is a USB3 device or port',
 764     },
 765 };
 766
 767 my $usbdesc = {
 768     optional => 1,
 769     type => 'string', format => $usb_fmt,
 770     description => <<EODESCR,
 771 Configure an USB device (n is 0 to 4). This can be used to
 772 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 773
 774 'bus-port(.port)*' (decimal numbers) or
 775 'vendor_id:product_id' (hexadeciaml numbers) or
 776 'spice'
 777
 778 You can use the 'lsusb -t' command to list existing usb devices.
 779
 780 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 781
 782 The value 'spice' can be used to add a usb redirection devices for spice.
 783
 784 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 785
 786 EODESCR
 787 };
 788 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 789
 790 my $hostpcidesc = {
 791         optional => 1,
 792         type => 'string', format => 'pve-qm-hostpci',
 793         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 794         description => <<EODESCR,
 795 Map host pci devices. HOSTPCIDEVICE syntax is:
 796
 797 'bus:dev.func' (hexadecimal numbers)
 798
 799 You can us the 'lspci' command to list existing pci devices.
 800
 801 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 802
 803 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 804
 805 Experimental: user reported problems with this option.
 806 EODESCR
 807 };
 808 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 809
 810 my $serialdesc = {
 811         optional => 1,
 812         type => 'string',
 813         pattern => '(/dev/.+|socket)',
 814         description =>  <<EODESCR,
 815 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 816
 817 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 818
 819 Experimental: user reported problems with this option.
 820 EODESCR
 821 };
 822
 823 my $paralleldesc= {
 824         optional => 1,
 825         type => 'string',
 826         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 827         description =>  <<EODESCR,
 828 Map host parallel devices (n is 0 to 2).
 829
 830 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 831
 832 Experimental: user reported problems with this option.
 833 EODESCR
 834 };
 835
 836 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 837     $confdesc->{"parallel$i"} = $paralleldesc;
 838 }
 839
 840 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 841     $confdesc->{"serial$i"} = $serialdesc;
 842 }
 843
 844 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 845     $confdesc->{"hostpci$i"} = $hostpcidesc;
 846 }
 847
 848 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 849     $drivename_hash->{"ide$i"} = 1;
 850     $confdesc->{"ide$i"} = $idedesc;
 851 }
 852
 853 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 854     $drivename_hash->{"sata$i"} = 1;
 855     $confdesc->{"sata$i"} = $satadesc;
 856 }
 857
 858 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 859     $drivename_hash->{"scsi$i"} = 1;
 860     $confdesc->{"scsi$i"} = $scsidesc ;
 861 }
 862
 863 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 864     $drivename_hash->{"virtio$i"} = 1;
 865     $confdesc->{"virtio$i"} = $virtiodesc;
 866 }
 867
 868 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 869     $confdesc->{"usb$i"} = $usbdesc;
 870 }
 871
 872 my $unuseddesc = {
 873     optional => 1,
 874     type => 'string', format => 'pve-volume-id',
 875     description => "Reference to unused volumes.",
 876 };
 877
 878 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 879     $confdesc->{"unused$i"} = $unuseddesc;
 880 }
 881
 882 my $kvm_api_version = 0;
 883
 884 sub kvm_version {
 885
 886     return $kvm_api_version if $kvm_api_version;
 887
 888     my $fh = IO::File->new("</dev/kvm") ||
 889         return 0;
 890
 891     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 892         $kvm_api_version = $v;
 893     }
 894
 895     $fh->close();
 896
 897     return  $kvm_api_version;
 898 }
 899
 900 my $kvm_user_version;
 901
 902 sub kvm_user_version {
 903
 904     return $kvm_user_version if $kvm_user_version;
 905
 906     $kvm_user_version = 'unknown';
 907
 908     my $code = sub {
 909         my $line = shift;
 910         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 911             $kvm_user_version = $2;
 912         }
 913     };
 914
 915     eval { run_command("kvm -version", outfunc => $code); };
 916     warn $@ if $@;
 917
 918     return $kvm_user_version;
 919
 920 }
 921
 922 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 923
 924 sub valid_drive_names {
 925     # order is important - used to autoselect boot disk
 926     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 927             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 928             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 929             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 930 }
 931
 932 sub is_valid_drivename {
 933     my $dev = shift;
 934
 935     return defined($drivename_hash->{$dev});
 936 }
 937
 938 sub option_exists {
 939     my $key = shift;
 940     return defined($confdesc->{$key});
 941 }
 942
 943 sub nic_models {
 944     return $nic_model_list;
 945 }
 946
 947 sub os_list_description {
 948
 949     return {
 950         other => 'Other',
 951         wxp => 'Windows XP',
 952         w2k => 'Windows 2000',
 953         w2k3 =>, 'Windows 2003',
 954         w2k8 => 'Windows 2008',
 955         wvista => 'Windows Vista',
 956         win7 => 'Windows 7',
 957         win8 => 'Windows 8/2012',
 958         l24 => 'Linux 2.4',
 959         l26 => 'Linux 2.6',
 960     };
 961 }
 962
 963 my $cdrom_path;
 964
 965 sub get_cdrom_path {
 966
 967     return  $cdrom_path if $cdrom_path;
 968
 969     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 970     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 971     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 972 }
 973
 974 sub get_iso_path {
 975     my ($storecfg, $vmid, $cdrom) = @_;
 976
 977     if ($cdrom eq 'cdrom') {
 978         return get_cdrom_path();
 979     } elsif ($cdrom eq 'none') {
 980         return '';
 981     } elsif ($cdrom =~ m|^/|) {
 982         return $cdrom;
 983     } else {
 984         return PVE::Storage::path($storecfg, $cdrom);
 985     }
 986 }
 987
 988 # try to convert old style file names to volume IDs
 989 sub filename_to_volume_id {
 990     my ($vmid, $file, $media) = @_;
 991
 992     if (!($file eq 'none' || $file eq 'cdrom' ||
 993           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 994
 995         return undef if $file =~ m|/|;
 996
 997         if ($media && $media eq 'cdrom') {
 998             $file = "local:iso/$file";
 999         } else {
1000             $file = "local:$vmid/$file";
1001         }
1002     }
1003
1004     return $file;
1005 }
1006
1007 sub verify_media_type {
1008     my ($opt, $vtype, $media) = @_;
1009
1010     return if !$media;
1011
1012     my $etype;
1013     if ($media eq 'disk') {
1014         $etype = 'images';
1015     } elsif ($media eq 'cdrom') {
1016         $etype = 'iso';
1017     } else {
1018         die "internal error";
1019     }
1020
1021     return if ($vtype eq $etype);
1022
1023     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1024 }
1025
1026 sub cleanup_drive_path {
1027     my ($opt, $storecfg, $drive) = @_;
1028
1029     # try to convert filesystem paths to volume IDs
1030
1031     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1032         ($drive->{file} !~ m|^/dev/.+|) &&
1033         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1034         ($drive->{file} !~ m/^\d+$/)) {
1035         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1036         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1037         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1038         verify_media_type($opt, $vtype, $drive->{media});
1039         $drive->{file} = $volid;
1040     }
1041
1042     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1043 }
1044
1045 sub parse_hotplug_features {
1046     my ($data) = @_;
1047
1048     my $res = {};
1049
1050     return $res if $data eq '0';
1051
1052     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1053
1054     foreach my $feature (PVE::Tools::split_list($data)) {
1055         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1056             $res->{$1} = 1;
1057         } else {
1058             warn "ignoring unknown hotplug feature '$feature'\n";
1059         }
1060     }
1061     return $res;
1062 }
1063
1064 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1065 sub pve_verify_hotplug_features {
1066     my ($value, $noerr) = @_;
1067
1068     return $value if parse_hotplug_features($value);
1069
1070     return undef if $noerr;
1071
1072     die "unable to parse hotplug option\n";
1073 }
1074
1075 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1076 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1077 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1078 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1079 #        [,iothread=on][,serial=serial][,model=model]
1080
1081 sub parse_drive {
1082     my ($key, $data) = @_;
1083
1084     my ($interface, $index);
1085
1086     if ($key =~ m/^([^\d]+)(\d+)$/) {
1087         $interface = $1;
1088         $index = $2;
1089     } else {
1090         return undef;
1091     }
1092
1093     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1094                                      : $confdesc->{$key}->{format};
1095     if (!$desc) {
1096         warn "invalid drive key: $key\n";
1097         return undef;
1098     }
1099     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1100     return undef if !$res;
1101     $res->{interface} = $interface;
1102     $res->{index} = $index;
1103
1104     my $error = 0;
1105     foreach my $opt (qw(bps bps_rd bps_wr)) {
1106         if (my $bps = defined(delete $res->{$opt})) {
1107             if (defined($res->{"m$opt"})) {
1108                 warn "both $opt and m$opt specified\n";
1109                 ++$error;
1110                 next;
1111             }
1112             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1113         }
1114     }
1115     return undef if $error;
1116
1117     return undef if $res->{mbps_rd} && $res->{mbps};
1118     return undef if $res->{mbps_wr} && $res->{mbps};
1119     return undef if $res->{iops_rd} && $res->{iops};
1120     return undef if $res->{iops_wr} && $res->{iops};
1121
1122     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1123         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1124         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1125         return undef if $res->{interface} eq 'virtio';
1126     }
1127
1128     if (my $size = $res->{size}) {
1129         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1130     }
1131
1132     return $res;
1133 }
1134
1135 sub print_drive {
1136     my ($vmid, $drive) = @_;
1137     my $data = { %$drive };
1138     delete $data->{$_} for qw(index interface);
1139     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1140 }
1141
1142 sub scsi_inquiry {
1143     my($fh, $noerr) = @_;
1144
1145     my $SG_IO = 0x2285;
1146     my $SG_GET_VERSION_NUM = 0x2282;
1147
1148     my $versionbuf = "\x00" x 8;
1149     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1150     if (!$ret) {
1151         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1152         return undef;
1153     }
1154     my $version = unpack("I", $versionbuf);
1155     if ($version < 30000) {
1156         die "scsi generic interface too old\n"  if !$noerr;
1157         return undef;
1158     }
1159
1160     my $buf = "\x00" x 36;
1161     my $sensebuf = "\x00" x 8;
1162     my $cmd = pack("C x3 C x1", 0x12, 36);
1163
1164     # see /usr/include/scsi/sg.h
1165     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1166
1167     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1168                       length($sensebuf), 0, length($buf), $buf,
1169                       $cmd, $sensebuf, 6000);
1170
1171     $ret = ioctl($fh, $SG_IO, $packet);
1172     if (!$ret) {
1173         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1174         return undef;
1175     }
1176
1177     my @res = unpack($sg_io_hdr_t, $packet);
1178     if ($res[17] || $res[18]) {
1179         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1180         return undef;
1181     }
1182
1183     my $res = {};
1184     (my $byte0, my $byte1, $res->{vendor},
1185      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1186
1187     $res->{removable} = $byte1 & 128 ? 1 : 0;
1188     $res->{type} = $byte0 & 31;
1189
1190     return $res;
1191 }
1192
1193 sub path_is_scsi {
1194     my ($path) = @_;
1195
1196     my $fh = IO::File->new("+<$path") || return undef;
1197     my $res = scsi_inquiry($fh, 1);
1198     close($fh);
1199
1200     return $res;
1201 }
1202
1203 sub machine_type_is_q35 {
1204     my ($conf) = @_;
1205
1206     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1207 }
1208
1209 sub print_tabletdevice_full {
1210     my ($conf) = @_;
1211
1212     my $q35 = machine_type_is_q35($conf);
1213
1214     # we use uhci for old VMs because tablet driver was buggy in older qemu
1215     my $usbbus = $q35 ? "ehci" : "uhci";
1216
1217     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1218 }
1219
1220 sub print_drivedevice_full {
1221     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1222
1223     my $device = '';
1224     my $maxdev = 0;
1225
1226     if ($drive->{interface} eq 'virtio') {
1227         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1228         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1229         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1230     } elsif ($drive->{interface} eq 'scsi') {
1231
1232         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1233         my $unit = $drive->{index} % $maxdev;
1234         my $devicetype = 'hd';
1235         my $path = '';
1236         if (drive_is_cdrom($drive)) {
1237             $devicetype = 'cd';
1238         } else {
1239             if ($drive->{file} =~ m|^/|) {
1240                 $path = $drive->{file};
1241                 if (my $info = path_is_scsi($path)) {
1242                     if ($info->{type} == 0) {
1243                         $devicetype = 'block';
1244                     } elsif ($info->{type} == 1) { # tape
1245                         $devicetype = 'generic';
1246                     }
1247                 }
1248             } else {
1249                  $path = PVE::Storage::path($storecfg, $drive->{file});
1250             }
1251
1252             if($path =~ m/^iscsi\:\/\//){
1253                 $devicetype = 'generic';
1254             }
1255         }
1256
1257         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1258             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1259         } else {
1260             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1261         }
1262
1263     } elsif ($drive->{interface} eq 'ide'){
1264         $maxdev = 2;
1265         my $controller = int($drive->{index} / $maxdev);
1266         my $unit = $drive->{index} % $maxdev;
1267         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1268
1269         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1270         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1271             $model = URI::Escape::uri_unescape($model);
1272             $device .= ",model=$model";
1273         }
1274     } elsif ($drive->{interface} eq 'sata'){
1275         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1276         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1277         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1278     } elsif ($drive->{interface} eq 'usb') {
1279         die "implement me";
1280         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1281     } else {
1282         die "unsupported interface type";
1283     }
1284
1285     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1286
1287     return $device;
1288 }
1289
1290 sub get_initiator_name {
1291     my $initiator;
1292
1293     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1294     while (defined(my $line = <$fh>)) {
1295         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1296         $initiator = $1;
1297         last;
1298     }
1299     $fh->close();
1300
1301     return $initiator;
1302 }
1303
1304 sub print_drive_full {
1305     my ($storecfg, $vmid, $drive) = @_;
1306
1307     my $path;
1308     my $volid = $drive->{file};
1309     my $format;
1310
1311     if (drive_is_cdrom($drive)) {
1312         $path = get_iso_path($storecfg, $vmid, $volid);
1313     } else {
1314         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1315         if ($storeid) {
1316             $path = PVE::Storage::path($storecfg, $volid);
1317             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1318             $format = qemu_img_format($scfg, $volname);
1319         } else {
1320             $path = $volid;
1321             $format = "raw";
1322         }
1323    }
1324
1325     my $opts = '';
1326     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1327     foreach my $o (@qemu_drive_options) {
1328         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1329     }
1330     if (my $serial = $drive->{serial}) {
1331         $serial = URI::Escape::uri_unescape($serial);
1332         $opts .= ",serial=$serial";
1333     }
1334
1335     $opts .= ",format=$format" if $format && !$drive->{format};
1336
1337     foreach my $o (qw(bps bps_rd bps_wr)) {
1338         my $v = $drive->{"m$o"};
1339         $opts .= ",$o=" . int($v*1024*1024) if $v;
1340     }
1341
1342     my $cache_direct = 0;
1343
1344     if (my $cache = $drive->{cache}) {
1345         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1346     } elsif (!drive_is_cdrom($drive)) {
1347         $opts .= ",cache=none";
1348         $cache_direct = 1;
1349     }
1350
1351     # aio native works only with O_DIRECT
1352     if (!$drive->{aio}) {
1353         if($cache_direct) {
1354             $opts .= ",aio=native";
1355         } else {
1356             $opts .= ",aio=threads";
1357         }
1358     }
1359
1360     if (!drive_is_cdrom($drive)) {
1361         my $detectzeroes;
1362         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1363             $detectzeroes = 'off';
1364         } elsif ($drive->{discard}) {
1365             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1366         } else {
1367             # This used to be our default with discard not being specified:
1368             $detectzeroes = 'on';
1369         }
1370         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1371     }
1372
1373     my $pathinfo = $path ? "file=$path," : '';
1374
1375     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1376 }
1377
1378 sub print_netdevice_full {
1379     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1380
1381     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1382
1383     my $device = $net->{model};
1384     if ($net->{model} eq 'virtio') {
1385          $device = 'virtio-net-pci';
1386      };
1387
1388     my $pciaddr = print_pci_addr("$netid", $bridges);
1389     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1390     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1391         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1392         my $vectors = $net->{queues} * 2 + 2;
1393         $tmpstr .= ",vectors=$vectors,mq=on";
1394     }
1395     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1396
1397     if ($use_old_bios_files) {
1398         my $romfile;
1399         if ($device eq 'virtio-net-pci') {
1400             $romfile = 'pxe-virtio.rom';
1401         } elsif ($device eq 'e1000') {
1402             $romfile = 'pxe-e1000.rom';
1403         } elsif ($device eq 'ne2k') {
1404             $romfile = 'pxe-ne2k_pci.rom';
1405         } elsif ($device eq 'pcnet') {
1406             $romfile = 'pxe-pcnet.rom';
1407         } elsif ($device eq 'rtl8139') {
1408             $romfile = 'pxe-rtl8139.rom';
1409         }
1410         $tmpstr .= ",romfile=$romfile" if $romfile;
1411     }
1412
1413     return $tmpstr;
1414 }
1415
1416 sub print_netdev_full {
1417     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1418
1419     my $i = '';
1420     if ($netid =~ m/^net(\d+)$/) {
1421         $i = int($1);
1422     }
1423
1424     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1425
1426     my $ifname = "tap${vmid}i$i";
1427
1428     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1429     die "interface name '$ifname' is too long (max 15 character)\n"
1430         if length($ifname) >= 16;
1431
1432     my $vhostparam = '';
1433     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1434
1435     my $vmname = $conf->{name} || "vm$vmid";
1436
1437     my $netdev = "";
1438     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1439
1440     if ($net->{bridge}) {
1441         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1442     } else {
1443         $netdev = "type=user,id=$netid,hostname=$vmname";
1444     }
1445
1446     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1447
1448     return $netdev;
1449 }
1450
1451 sub drive_is_cdrom {
1452     my ($drive) = @_;
1453
1454     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1455
1456 }
1457
1458 sub parse_numa {
1459     my ($data) = @_;
1460
1461     my $res = {};
1462
1463     foreach my $kvp (split(/,/, $data)) {
1464
1465         if ($kvp =~ m/^memory=(\S+)$/) {
1466             $res->{memory} = $1;
1467         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1468             $res->{policy} = $1;
1469         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1470             $res->{cpus}->{start} = $1;
1471             $res->{cpus}->{end} = $3;
1472         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1473             $res->{hostnodes}->{start} = $1;
1474             $res->{hostnodes}->{end} = $3;
1475         } else {
1476             return undef;
1477         }
1478     }
1479
1480     return $res;
1481 }
1482
1483 sub parse_hostpci {
1484     my ($value) = @_;
1485
1486     return undef if !$value;
1487
1488
1489     my @list = split(/,/, $value);
1490     my $found;
1491
1492     my $res = {};
1493     foreach my $kv (@list) {
1494
1495         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1496             $found = 1;
1497             if(defined($4)){
1498                 push @{$res->{pciid}}, { id => $2 , function => $4};
1499
1500             }else{
1501                 my $pcidevices = lspci($2);
1502                 $res->{pciid} = $pcidevices->{$2};
1503             }
1504         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1505             $res->{rombar} = $1;
1506         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1507             $res->{'x-vga'} = $1;
1508         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1509             $res->{pcie} = 1 if $1 == 1;
1510         } else {
1511             warn "unknown hostpci setting '$kv'\n";
1512         }
1513     }
1514
1515     return undef if !$found;
1516
1517     return $res;
1518 }
1519
1520 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1521 sub parse_net {
1522     my ($data) = @_;
1523
1524     my $res = {};
1525
1526     foreach my $kvp (split(/,/, $data)) {
1527
1528         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1529             my $model = lc($1);
1530             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1531             $res->{model} = $model;
1532             $res->{macaddr} = $mac;
1533         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1534             $res->{bridge} = $1;
1535         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1536             $res->{queues} = $1;
1537         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1538             $res->{rate} = $1;
1539         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1540             $res->{tag} = $1;
1541         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1542             $res->{trunks} = $1;
1543         } elsif ($kvp =~ m/^firewall=([01])$/) {
1544             $res->{firewall} = $1;
1545         } elsif ($kvp =~ m/^link_down=([01])$/) {
1546             $res->{link_down} = $1;
1547         } else {
1548             return undef;
1549         }
1550
1551     }
1552
1553     return undef if !$res->{model};
1554
1555     return $res;
1556 }
1557
1558 sub print_net {
1559     my $net = shift;
1560
1561     my $res = "$net->{model}";
1562     $res .= "=$net->{macaddr}" if $net->{macaddr};
1563     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1564     $res .= ",rate=$net->{rate}" if $net->{rate};
1565     $res .= ",tag=$net->{tag}" if $net->{tag};
1566     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1567     $res .= ",firewall=1" if $net->{firewall};
1568     $res .= ",link_down=1" if $net->{link_down};
1569     $res .= ",queues=$net->{queues}" if $net->{queues};
1570
1571     return $res;
1572 }
1573
1574 sub add_random_macs {
1575     my ($settings) = @_;
1576
1577     foreach my $opt (keys %$settings) {
1578         next if $opt !~ m/^net(\d+)$/;
1579         my $net = parse_net($settings->{$opt});
1580         next if !$net;
1581         $settings->{$opt} = print_net($net);
1582     }
1583 }
1584
1585 sub vm_is_volid_owner {
1586     my ($storecfg, $vmid, $volid) = @_;
1587
1588     if ($volid !~  m|^/|) {
1589         my ($path, $owner);
1590         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1591         if ($owner && ($owner == $vmid)) {
1592             return 1;
1593         }
1594     }
1595
1596     return undef;
1597 }
1598
1599 sub split_flagged_list {
1600     my $text = shift || '';
1601     $text =~ s/[,;]/ /g;
1602     $text =~ s/^\s+//;
1603     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1604 }
1605
1606 sub join_flagged_list {
1607     my ($how, $lst) = @_;
1608     join $how, map { $lst->{$_} . $_ } keys %$lst;
1609 }
1610
1611 sub vmconfig_delete_pending_option {
1612     my ($conf, $key, $force) = @_;
1613
1614     delete $conf->{pending}->{$key};
1615     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1616     $pending_delete_hash->{$key} = $force ? '!' : '';
1617     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1618 }
1619
1620 sub vmconfig_undelete_pending_option {
1621     my ($conf, $key) = @_;
1622
1623     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1624     delete $pending_delete_hash->{$key};
1625
1626     if (%$pending_delete_hash) {
1627         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1628     } else {
1629         delete $conf->{pending}->{delete};
1630     }
1631 }
1632
1633 sub vmconfig_register_unused_drive {
1634     my ($storecfg, $vmid, $conf, $drive) = @_;
1635
1636     if (!drive_is_cdrom($drive)) {
1637         my $volid = $drive->{file};
1638         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1639             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1640         }
1641     }
1642 }
1643
1644 sub vmconfig_cleanup_pending {
1645     my ($conf) = @_;
1646
1647     # remove pending changes when nothing changed
1648     my $changes;
1649     foreach my $opt (keys %{$conf->{pending}}) {
1650         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1651             $changes = 1;
1652             delete $conf->{pending}->{$opt};
1653         }
1654     }
1655
1656     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1657     my $pending_delete_hash = {};
1658     while (my ($opt, $force) = each %$current_delete_hash) {
1659         if (defined($conf->{$opt})) {
1660             $pending_delete_hash->{$opt} = $force;
1661         } else {
1662             $changes = 1;
1663         }
1664     }
1665
1666     if (%$pending_delete_hash) {
1667         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1668     } else {
1669         delete $conf->{pending}->{delete};
1670     }
1671
1672     return $changes;
1673 }
1674
1675 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1676 my $smbios1_fmt = {
1677     uuid => {
1678         type => 'string',
1679         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1680         format_description => 'UUID',
1681         optional => 1,
1682     },
1683     version => {
1684         type => 'string',
1685         pattern => '\S+',
1686         format_description => 'str',
1687         optional => 1,
1688     },
1689     serial => {
1690         type => 'string',
1691         pattern => '\S+',
1692         format_description => 'str',
1693         optional => 1,
1694     },
1695     manufacturer => {
1696         type => 'string',
1697         pattern => '\S+',
1698         format_description => 'name',
1699         optional => 1,
1700     },
1701     product => {
1702         type => 'string',
1703         pattern => '\S+',
1704         format_description => 'name',
1705         optional => 1,
1706     },
1707     sku => {
1708         type => 'string',
1709         pattern => '\S+',
1710         format_description => 'str',
1711         optional => 1,
1712     },
1713     family => {
1714         type => 'string',
1715         pattern => '\S+',
1716         format_description => 'str',
1717         optional => 1,
1718     },
1719 };
1720
1721 sub parse_smbios1 {
1722     my ($data) = @_;
1723
1724     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1725     warn $@ if $@;
1726     return $res;
1727 }
1728
1729 sub print_smbios1 {
1730     my ($smbios1) = @_;
1731     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1732 }
1733
1734 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1735
1736 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1737 sub verify_bootdisk {
1738     my ($value, $noerr) = @_;
1739
1740     return $value if is_valid_drivename($value);
1741
1742     return undef if $noerr;
1743
1744     die "invalid boot disk '$value'\n";
1745 }
1746
1747 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1748 sub verify_numa {
1749     my ($value, $noerr) = @_;
1750
1751     return $value if parse_numa($value);
1752
1753     return undef if $noerr;
1754
1755     die "unable to parse numa options\n";
1756 }
1757
1758 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1759 sub verify_net {
1760     my ($value, $noerr) = @_;
1761
1762     return $value if parse_net($value);
1763
1764     return undef if $noerr;
1765
1766     die "unable to parse network options\n";
1767 }
1768
1769 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1770 sub verify_hostpci {
1771     my ($value, $noerr) = @_;
1772
1773     return $value if parse_hostpci($value);
1774
1775     return undef if $noerr;
1776
1777     die "unable to parse pci id\n";
1778 }
1779
1780 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1781 sub verify_watchdog {
1782     my ($value, $noerr) = @_;
1783
1784     return $value if parse_watchdog($value);
1785
1786     return undef if $noerr;
1787
1788     die "unable to parse watchdog options\n";
1789 }
1790
1791 sub parse_watchdog {
1792     my ($value) = @_;
1793
1794     return undef if !$value;
1795
1796     my $res = {};
1797
1798     foreach my $p (split(/,/, $value)) {
1799         next if $p =~ m/^\s*$/;
1800
1801         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1802             $res->{model} = $2;
1803         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1804             $res->{action} = $2;
1805         } else {
1806             return undef;
1807         }
1808     }
1809
1810     return $res;
1811 }
1812
1813 sub parse_usb_device {
1814     my ($value) = @_;
1815
1816     return undef if !$value;
1817
1818     my $res = {};
1819     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1820         $res->{vendorid} = $2;
1821         $res->{productid} = $4;
1822     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1823         $res->{hostbus} = $1;
1824         $res->{hostport} = $2;
1825     } elsif ($value =~ m/^spice$/i) {
1826         $res->{spice} = 1;
1827     } else {
1828         return undef;
1829     }
1830
1831     return $res;
1832 }
1833
1834 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1835 sub verify_usb_device {
1836     my ($value, $noerr) = @_;
1837
1838     return $value if parse_usb_device($value);
1839
1840     return undef if $noerr;
1841
1842     die "unable to parse usb device\n";
1843 }
1844
1845 # add JSON properties for create and set function
1846 sub json_config_properties {
1847     my $prop = shift;
1848
1849     foreach my $opt (keys %$confdesc) {
1850         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1851         $prop->{$opt} = $confdesc->{$opt};
1852     }
1853
1854     return $prop;
1855 }
1856
1857 sub check_type {
1858     my ($key, $value) = @_;
1859
1860     die "unknown setting '$key'\n" if !$confdesc->{$key};
1861
1862     my $type = $confdesc->{$key}->{type};
1863
1864     if (!defined($value)) {
1865         die "got undefined value\n";
1866     }
1867
1868     if ($value =~ m/[\n\r]/) {
1869         die "property contains a line feed\n";
1870     }
1871
1872     if ($type eq 'boolean') {
1873         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1874         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1875         die "type check ('boolean') failed - got '$value'\n";
1876     } elsif ($type eq 'integer') {
1877         return int($1) if $value =~ m/^(\d+)$/;
1878         die "type check ('integer') failed - got '$value'\n";
1879     } elsif ($type eq 'number') {
1880         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1881         die "type check ('number') failed - got '$value'\n";
1882     } elsif ($type eq 'string') {
1883         if (my $fmt = $confdesc->{$key}->{format}) {
1884             if ($fmt eq 'pve-qm-drive') {
1885                 # special case - we need to pass $key to parse_drive()
1886                 my $drive = parse_drive($key, $value);
1887                 return $value if $drive;
1888                 die "unable to parse drive options\n";
1889             }
1890             PVE::JSONSchema::check_format($fmt, $value);
1891             return $value;
1892         }
1893         $value =~ s/^\"(.*)\"$/$1/;
1894         return $value;
1895     } else {
1896         die "internal error"
1897     }
1898 }
1899
1900 sub check_iommu_support{
1901     #fixme : need to check IOMMU support
1902     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1903
1904     my $iommu=1;
1905     return $iommu;
1906
1907 }
1908
1909 sub touch_config {
1910     my ($vmid) = @_;
1911
1912     my $conf = PVE::QemuConfig->config_file($vmid);
1913     utime undef, undef, $conf;
1914 }
1915
1916 sub destroy_vm {
1917     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1918
1919     my $conffile = PVE::QemuConfig->config_file($vmid);
1920
1921     my $conf = PVE::QemuConfig->load_config($vmid);
1922
1923     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1924
1925     # only remove disks owned by this VM
1926     foreach_drive($conf, sub {
1927         my ($ds, $drive) = @_;
1928
1929         return if drive_is_cdrom($drive);
1930
1931         my $volid = $drive->{file};
1932
1933         return if !$volid || $volid =~ m|^/|;
1934
1935         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1936         return if !$path || !$owner || ($owner != $vmid);
1937
1938         PVE::Storage::vdisk_free($storecfg, $volid);
1939     });
1940
1941     if ($keep_empty_config) {
1942         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1943     } else {
1944         unlink $conffile;
1945     }
1946
1947     # also remove unused disk
1948     eval {
1949         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1950
1951         eval {
1952             PVE::Storage::foreach_volid($dl, sub {
1953                 my ($volid, $sid, $volname, $d) = @_;
1954                 PVE::Storage::vdisk_free($storecfg, $volid);
1955             });
1956         };
1957         warn $@ if $@;
1958
1959     };
1960     warn $@ if $@;
1961 }
1962
1963 sub parse_vm_config {
1964     my ($filename, $raw) = @_;
1965
1966     return undef if !defined($raw);
1967
1968     my $res = {
1969         digest => Digest::SHA::sha1_hex($raw),
1970         snapshots => {},
1971         pending => {},
1972     };
1973
1974     $filename =~ m|/qemu-server/(\d+)\.conf$|
1975         || die "got strange filename '$filename'";
1976
1977     my $vmid = $1;
1978
1979     my $conf = $res;
1980     my $descr;
1981     my $section = '';
1982
1983     my @lines = split(/\n/, $raw);
1984     foreach my $line (@lines) {
1985         next if $line =~ m/^\s*$/;
1986
1987         if ($line =~ m/^\[PENDING\]\s*$/i) {
1988             $section = 'pending';
1989             if (defined($descr)) {
1990                 $descr =~ s/\s+$//;
1991                 $conf->{description} = $descr;
1992             }
1993             $descr = undef;
1994             $conf = $res->{$section} = {};
1995             next;
1996
1997         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1998             $section = $1;
1999             if (defined($descr)) {
2000                 $descr =~ s/\s+$//;
2001                 $conf->{description} = $descr;
2002             }
2003             $descr = undef;
2004             $conf = $res->{snapshots}->{$section} = {};
2005             next;
2006         }
2007
2008         if ($line =~ m/^\#(.*)\s*$/) {
2009             $descr = '' if !defined($descr);
2010             $descr .= PVE::Tools::decode_text($1) . "\n";
2011             next;
2012         }
2013
2014         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2015             $descr = '' if !defined($descr);
2016             $descr .= PVE::Tools::decode_text($2);
2017         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2018             $conf->{snapstate} = $1;
2019         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2020             my $key = $1;
2021             my $value = $2;
2022             $conf->{$key} = $value;
2023         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2024             my $value = $1;
2025             if ($section eq 'pending') {
2026                 $conf->{delete} = $value; # we parse this later
2027             } else {
2028                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2029             }
2030         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2031             my $key = $1;
2032             my $value = $2;
2033             eval { $value = check_type($key, $value); };
2034             if ($@) {
2035                 warn "vm $vmid - unable to parse value of '$key' - $@";
2036             } else {
2037                 my $fmt = $confdesc->{$key}->{format};
2038                 if ($fmt && $fmt eq 'pve-qm-drive') {
2039                     my $v = parse_drive($key, $value);
2040                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2041                         $v->{file} = $volid;
2042                         $value = print_drive($vmid, $v);
2043                     } else {
2044                         warn "vm $vmid - unable to parse value of '$key'\n";
2045                         next;
2046                     }
2047                 }
2048
2049                 if ($key eq 'cdrom') {
2050                     $conf->{ide2} = $value;
2051                 } else {
2052                     $conf->{$key} = $value;
2053                 }
2054             }
2055         }
2056     }
2057
2058     if (defined($descr)) {
2059         $descr =~ s/\s+$//;
2060         $conf->{description} = $descr;
2061     }
2062     delete $res->{snapstate}; # just to be sure
2063
2064     return $res;
2065 }
2066
2067 sub write_vm_config {
2068     my ($filename, $conf) = @_;
2069
2070     delete $conf->{snapstate}; # just to be sure
2071
2072     if ($conf->{cdrom}) {
2073         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2074         $conf->{ide2} = $conf->{cdrom};
2075         delete $conf->{cdrom};
2076     }
2077
2078     # we do not use 'smp' any longer
2079     if ($conf->{sockets}) {
2080         delete $conf->{smp};
2081     } elsif ($conf->{smp}) {
2082         $conf->{sockets} = $conf->{smp};
2083         delete $conf->{cores};
2084         delete $conf->{smp};
2085     }
2086
2087     my $used_volids = {};
2088
2089     my $cleanup_config = sub {
2090         my ($cref, $pending, $snapname) = @_;
2091
2092         foreach my $key (keys %$cref) {
2093             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2094                 $key eq 'snapstate' || $key eq 'pending';
2095             my $value = $cref->{$key};
2096             if ($key eq 'delete') {
2097                 die "propertry 'delete' is only allowed in [PENDING]\n"
2098                     if !$pending;
2099                 # fixme: check syntax?
2100                 next;
2101             }
2102             eval { $value = check_type($key, $value); };
2103             die "unable to parse value of '$key' - $@" if $@;
2104
2105             $cref->{$key} = $value;
2106
2107             if (!$snapname && is_valid_drivename($key)) {
2108                 my $drive = parse_drive($key, $value);
2109                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2110             }
2111         }
2112     };
2113
2114     &$cleanup_config($conf);
2115
2116     &$cleanup_config($conf->{pending}, 1);
2117
2118     foreach my $snapname (keys %{$conf->{snapshots}}) {
2119         die "internal error" if $snapname eq 'pending';
2120         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2121     }
2122
2123     # remove 'unusedX' settings if we re-add a volume
2124     foreach my $key (keys %$conf) {
2125         my $value = $conf->{$key};
2126         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2127             delete $conf->{$key};
2128         }
2129     }
2130
2131     my $generate_raw_config = sub {
2132         my ($conf, $pending) = @_;
2133
2134         my $raw = '';
2135
2136         # add description as comment to top of file
2137         if (defined(my $descr = $conf->{description})) {
2138             if ($descr) {
2139                 foreach my $cl (split(/\n/, $descr)) {
2140                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2141                 }
2142             } else {
2143                 $raw .= "#\n" if $pending;
2144             }
2145         }
2146
2147         foreach my $key (sort keys %$conf) {
2148             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2149             $raw .= "$key: $conf->{$key}\n";
2150         }
2151         return $raw;
2152     };
2153
2154     my $raw = &$generate_raw_config($conf);
2155
2156     if (scalar(keys %{$conf->{pending}})){
2157         $raw .= "\n[PENDING]\n";
2158         $raw .= &$generate_raw_config($conf->{pending}, 1);
2159     }
2160
2161     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2162         $raw .= "\n[$snapname]\n";
2163         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2164     }
2165
2166     return $raw;
2167 }
2168
2169 sub load_defaults {
2170
2171     my $res = {};
2172
2173     # we use static defaults from our JSON schema configuration
2174     foreach my $key (keys %$confdesc) {
2175         if (defined(my $default = $confdesc->{$key}->{default})) {
2176             $res->{$key} = $default;
2177         }
2178     }
2179
2180     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2181     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2182
2183     return $res;
2184 }
2185
2186 sub config_list {
2187     my $vmlist = PVE::Cluster::get_vmlist();
2188     my $res = {};
2189     return $res if !$vmlist || !$vmlist->{ids};
2190     my $ids = $vmlist->{ids};
2191
2192     foreach my $vmid (keys %$ids) {
2193         my $d = $ids->{$vmid};
2194         next if !$d->{node} || $d->{node} ne $nodename;
2195         next if !$d->{type} || $d->{type} ne 'qemu';
2196         $res->{$vmid}->{exists} = 1;
2197     }
2198     return $res;
2199 }
2200
2201 # test if VM uses local resources (to prevent migration)
2202 sub check_local_resources {
2203     my ($conf, $noerr) = @_;
2204
2205     my $loc_res = 0;
2206
2207     $loc_res = 1 if $conf->{hostusb}; # old syntax
2208     $loc_res = 1 if $conf->{hostpci}; # old syntax
2209
2210     foreach my $k (keys %$conf) {
2211         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2212         # sockets are safe: they will recreated be on the target side post-migrate
2213         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2214         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2215     }
2216
2217     die "VM uses local resources\n" if $loc_res && !$noerr;
2218
2219     return $loc_res;
2220 }
2221
2222 # check if used storages are available on all nodes (use by migrate)
2223 sub check_storage_availability {
2224     my ($storecfg, $conf, $node) = @_;
2225
2226     foreach_drive($conf, sub {
2227         my ($ds, $drive) = @_;
2228
2229         my $volid = $drive->{file};
2230         return if !$volid;
2231
2232         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2233         return if !$sid;
2234
2235         # check if storage is available on both nodes
2236         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2237         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2238    });
2239 }
2240
2241 # list nodes where all VM images are available (used by has_feature API)
2242 sub shared_nodes {
2243     my ($conf, $storecfg) = @_;
2244
2245     my $nodelist = PVE::Cluster::get_nodelist();
2246     my $nodehash = { map { $_ => 1 } @$nodelist };
2247     my $nodename = PVE::INotify::nodename();
2248
2249     foreach_drive($conf, sub {
2250         my ($ds, $drive) = @_;
2251
2252         my $volid = $drive->{file};
2253         return if !$volid;
2254
2255         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2256         if ($storeid) {
2257             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2258             if ($scfg->{disable}) {
2259                 $nodehash = {};
2260             } elsif (my $avail = $scfg->{nodes}) {
2261                 foreach my $node (keys %$nodehash) {
2262                     delete $nodehash->{$node} if !$avail->{$node};
2263                 }
2264             } elsif (!$scfg->{shared}) {
2265                 foreach my $node (keys %$nodehash) {
2266                     delete $nodehash->{$node} if $node ne $nodename
2267                 }
2268             }
2269         }
2270     });
2271
2272     return $nodehash
2273 }
2274
2275 sub check_cmdline {
2276     my ($pidfile, $pid) = @_;
2277
2278     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2279     if (defined($fh)) {
2280         my $line = <$fh>;
2281         $fh->close;
2282         return undef if !$line;
2283         my @param = split(/\0/, $line);
2284
2285         my $cmd = $param[0];
2286         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2287
2288         for (my $i = 0; $i < scalar (@param); $i++) {
2289             my $p = $param[$i];
2290             next if !$p;
2291             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2292                 my $p = $param[$i+1];
2293                 return 1 if $p && ($p eq $pidfile);
2294                 return undef;
2295             }
2296         }
2297     }
2298     return undef;
2299 }
2300
2301 sub check_running {
2302     my ($vmid, $nocheck, $node) = @_;
2303
2304     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2305
2306     die "unable to find configuration file for VM $vmid - no such machine\n"
2307         if !$nocheck && ! -f $filename;
2308
2309     my $pidfile = pidfile_name($vmid);
2310
2311     if (my $fd = IO::File->new("<$pidfile")) {
2312         my $st = stat($fd);
2313         my $line = <$fd>;
2314         close($fd);
2315
2316         my $mtime = $st->mtime;
2317         if ($mtime > time()) {
2318             warn "file '$filename' modified in future\n";
2319         }
2320
2321         if ($line =~ m/^(\d+)$/) {
2322             my $pid = $1;
2323             if (check_cmdline($pidfile, $pid)) {
2324                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2325                     return $pid;
2326                 }
2327             }
2328         }
2329     }
2330
2331     return undef;
2332 }
2333
2334 sub vzlist {
2335
2336     my $vzlist = config_list();
2337
2338     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2339
2340     while (defined(my $de = $fd->read)) {
2341         next if $de !~ m/^(\d+)\.pid$/;
2342         my $vmid = $1;
2343         next if !defined($vzlist->{$vmid});
2344         if (my $pid = check_running($vmid)) {
2345             $vzlist->{$vmid}->{pid} = $pid;
2346         }
2347     }
2348
2349     return $vzlist;
2350 }
2351
2352 sub disksize {
2353     my ($storecfg, $conf) = @_;
2354
2355     my $bootdisk = $conf->{bootdisk};
2356     return undef if !$bootdisk;
2357     return undef if !is_valid_drivename($bootdisk);
2358
2359     return undef if !$conf->{$bootdisk};
2360
2361     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2362     return undef if !defined($drive);
2363
2364     return undef if drive_is_cdrom($drive);
2365
2366     my $volid = $drive->{file};
2367     return undef if !$volid;
2368
2369     return $drive->{size};
2370 }
2371
2372 my $last_proc_pid_stat;
2373
2374 # get VM status information
2375 # This must be fast and should not block ($full == false)
2376 # We only query KVM using QMP if $full == true (this can be slow)
2377 sub vmstatus {
2378     my ($opt_vmid, $full) = @_;
2379
2380     my $res = {};
2381
2382     my $storecfg = PVE::Storage::config();
2383
2384     my $list = vzlist();
2385     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2386
2387     my $cpucount = $cpuinfo->{cpus} || 1;
2388
2389     foreach my $vmid (keys %$list) {
2390         next if $opt_vmid && ($vmid ne $opt_vmid);
2391
2392         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2393         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2394
2395         my $d = {};
2396         $d->{pid} = $list->{$vmid}->{pid};
2397
2398         # fixme: better status?
2399         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2400
2401         my $size = disksize($storecfg, $conf);
2402         if (defined($size)) {
2403             $d->{disk} = 0; # no info available
2404             $d->{maxdisk} = $size;
2405         } else {
2406             $d->{disk} = 0;
2407             $d->{maxdisk} = 0;
2408         }
2409
2410         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2411         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2412         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2413
2414         $d->{name} = $conf->{name} || "VM $vmid";
2415         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2416
2417         if ($conf->{balloon}) {
2418             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2419             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2420         }
2421
2422         $d->{uptime} = 0;
2423         $d->{cpu} = 0;
2424         $d->{mem} = 0;
2425
2426         $d->{netout} = 0;
2427         $d->{netin} = 0;
2428
2429         $d->{diskread} = 0;
2430         $d->{diskwrite} = 0;
2431
2432         $d->{template} = PVE::QemuConfig->is_template($conf);
2433
2434         $res->{$vmid} = $d;
2435     }
2436
2437     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2438     foreach my $dev (keys %$netdev) {
2439         next if $dev !~ m/^tap([1-9]\d*)i/;
2440         my $vmid = $1;
2441         my $d = $res->{$vmid};
2442         next if !$d;
2443
2444         $d->{netout} += $netdev->{$dev}->{receive};
2445         $d->{netin} += $netdev->{$dev}->{transmit};
2446
2447         if ($full) {
2448             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2449             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2450         }
2451
2452     }
2453
2454     my $ctime = gettimeofday;
2455
2456     foreach my $vmid (keys %$list) {
2457
2458         my $d = $res->{$vmid};
2459         my $pid = $d->{pid};
2460         next if !$pid;
2461
2462         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2463         next if !$pstat; # not running
2464
2465         my $used = $pstat->{utime} + $pstat->{stime};
2466
2467         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2468
2469         if ($pstat->{vsize}) {
2470             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2471         }
2472
2473         my $old = $last_proc_pid_stat->{$pid};
2474         if (!$old) {
2475             $last_proc_pid_stat->{$pid} = {
2476                 time => $ctime,
2477                 used => $used,
2478                 cpu => 0,
2479             };
2480             next;
2481         }
2482
2483         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2484
2485         if ($dtime > 1000) {
2486             my $dutime = $used -  $old->{used};
2487
2488             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2489             $last_proc_pid_stat->{$pid} = {
2490                 time => $ctime,
2491                 used => $used,
2492                 cpu => $d->{cpu},
2493             };
2494         } else {
2495             $d->{cpu} = $old->{cpu};
2496         }
2497     }
2498
2499     return $res if !$full;
2500
2501     my $qmpclient = PVE::QMPClient->new();
2502
2503     my $ballooncb = sub {
2504         my ($vmid, $resp) = @_;
2505
2506         my $info = $resp->{'return'};
2507         return if !$info->{max_mem};
2508
2509         my $d = $res->{$vmid};
2510
2511         # use memory assigned to VM
2512         $d->{maxmem} = $info->{max_mem};
2513         $d->{balloon} = $info->{actual};
2514
2515         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2516             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2517             $d->{freemem} = $info->{free_mem};
2518         }
2519
2520         $d->{ballooninfo} = $info;
2521     };
2522
2523     my $blockstatscb = sub {
2524         my ($vmid, $resp) = @_;
2525         my $data = $resp->{'return'} || [];
2526         my $totalrdbytes = 0;
2527         my $totalwrbytes = 0;
2528
2529         for my $blockstat (@$data) {
2530             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2531             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2532
2533             $blockstat->{device} =~ s/drive-//;
2534             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2535         }
2536         $res->{$vmid}->{diskread} = $totalrdbytes;
2537         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2538     };
2539
2540     my $statuscb = sub {
2541         my ($vmid, $resp) = @_;
2542
2543         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2544         # this fails if ballon driver is not loaded, so this must be
2545         # the last commnand (following command are aborted if this fails).
2546         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2547
2548         my $status = 'unknown';
2549         if (!defined($status = $resp->{'return'}->{status})) {
2550             warn "unable to get VM status\n";
2551             return;
2552         }
2553
2554         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2555     };
2556
2557     foreach my $vmid (keys %$list) {
2558         next if $opt_vmid && ($vmid ne $opt_vmid);
2559         next if !$res->{$vmid}->{pid}; # not running
2560         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2561     }
2562
2563     $qmpclient->queue_execute(undef, 1);
2564
2565     foreach my $vmid (keys %$list) {
2566         next if $opt_vmid && ($vmid ne $opt_vmid);
2567         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2568     }
2569
2570     return $res;
2571 }
2572
2573 sub foreach_dimm {
2574     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2575
2576     my $dimm_id = 0;
2577     my $current_size = 1024;
2578     my $dimm_size = 512;
2579     return if $current_size == $memory;
2580
2581     for (my $j = 0; $j < 8; $j++) {
2582         for (my $i = 0; $i < 32; $i++) {
2583             my $name = "dimm${dimm_id}";
2584             $dimm_id++;
2585             my $numanode = $i % $sockets;
2586             $current_size += $dimm_size;
2587             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2588             return  $current_size if $current_size >= $memory;
2589         }
2590         $dimm_size *= 2;
2591     }
2592 }
2593
2594 sub foreach_reverse_dimm {
2595     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2596
2597     my $dimm_id = 253;
2598     my $current_size = 4177920;
2599     my $dimm_size = 65536;
2600     return if $current_size == $memory;
2601
2602     for (my $j = 0; $j < 8; $j++) {
2603         for (my $i = 0; $i < 32; $i++) {
2604             my $name = "dimm${dimm_id}";
2605             $dimm_id--;
2606             my $numanode = $i % $sockets;
2607             $current_size -= $dimm_size;
2608             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2609             return  $current_size if $current_size <= $memory;
2610         }
2611         $dimm_size /= 2;
2612     }
2613 }
2614
2615 sub foreach_drive {
2616     my ($conf, $func) = @_;
2617
2618     foreach my $ds (valid_drive_names()) {
2619         next if !defined($conf->{$ds});
2620
2621         my $drive = parse_drive($ds, $conf->{$ds});
2622         next if !$drive;
2623
2624         &$func($ds, $drive);
2625     }
2626 }
2627
2628 sub foreach_volid {
2629     my ($conf, $func) = @_;
2630
2631     my $volhash = {};
2632
2633     my $test_volid = sub {
2634         my ($volid, $is_cdrom) = @_;
2635
2636         return if !$volid;
2637
2638         $volhash->{$volid} = $is_cdrom || 0;
2639     };
2640
2641     foreach_drive($conf, sub {
2642         my ($ds, $drive) = @_;
2643         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2644     });
2645
2646     foreach my $snapname (keys %{$conf->{snapshots}}) {
2647         my $snap = $conf->{snapshots}->{$snapname};
2648         &$test_volid($snap->{vmstate}, 0);
2649         foreach_drive($snap, sub {
2650             my ($ds, $drive) = @_;
2651             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2652         });
2653     }
2654
2655     foreach my $volid (keys %$volhash) {
2656         &$func($volid, $volhash->{$volid});
2657     }
2658 }
2659
2660 sub vga_conf_has_spice {
2661     my ($vga) = @_;
2662
2663     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2664
2665     return $1 || 1;
2666 }
2667
2668 sub config_to_command {
2669     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2670
2671     my $cmd = [];
2672     my $globalFlags = [];
2673     my $machineFlags = [];
2674     my $rtcFlags = [];
2675     my $cpuFlags = [];
2676     my $devices = [];
2677     my $pciaddr = '';
2678     my $bridges = {};
2679     my $kvmver = kvm_user_version();
2680     my $vernum = 0; # unknown
2681     my $ostype = $conf->{ostype};
2682     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2683         $vernum = $1*1000000+$2*1000;
2684     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2685         $vernum = $1*1000000+$2*1000+$3;
2686     }
2687
2688     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2689
2690     my $have_ovz = -f '/proc/vz/vestat';
2691
2692     my $q35 = machine_type_is_q35($conf);
2693     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2694     my $machine_type = $forcemachine || $conf->{machine};
2695     my $use_old_bios_files = undef;
2696     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2697
2698     my $cpuunits = defined($conf->{cpuunits}) ?
2699             $conf->{cpuunits} : $defaults->{cpuunits};
2700
2701     push @$cmd, '/usr/bin/systemd-run';
2702     push @$cmd, '--scope';
2703     push @$cmd, '--slice', "qemu";
2704     push @$cmd, '--unit', $vmid;
2705     # set KillMode=none, so that systemd don't kill those scopes
2706     # at shutdown (pve-manager service should stop the VMs instead)
2707     push @$cmd, '-p', "KillMode=none";
2708     push @$cmd, '-p', "CPUShares=$cpuunits";
2709     if ($conf->{cpulimit}) {
2710         my $cpulimit = int($conf->{cpulimit} * 100);
2711         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2712     }
2713
2714     push @$cmd, '/usr/bin/kvm';
2715
2716     push @$cmd, '-id', $vmid;
2717
2718     my $use_virtio = 0;
2719
2720     my $qmpsocket = qmp_socket($vmid);
2721     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2722     push @$cmd, '-mon', "chardev=qmp,mode=control";
2723
2724
2725     push @$cmd, '-pidfile' , pidfile_name($vmid);
2726
2727     push @$cmd, '-daemonize';
2728
2729     if ($conf->{smbios1}) {
2730         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2731     }
2732
2733     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2734         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2735         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2736         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2737         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2738         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2739         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2740     }
2741
2742     if ($q35) {
2743         # the q35 chipset support native usb2, so we enable usb controller
2744         # by default for this machine type
2745         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2746     } else {
2747         $pciaddr = print_pci_addr("piix3", $bridges);
2748         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2749
2750         my $use_usb2 = 0;
2751         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2752             next if !$conf->{"usb$i"};
2753             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2754             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2755             $use_usb2 = 1;
2756         }
2757         # include usb device config
2758         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2759     }
2760
2761     # add usb3 controller if needed
2762
2763     my $use_usb3 = 0;
2764     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2765         next if !$conf->{"usb$i"};
2766         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2767         next if !$d || !$d->{usb3};
2768         $use_usb3 = 1;
2769     }
2770
2771     $pciaddr = print_pci_addr("xhci", $bridges);
2772     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2773
2774     my $vga = $conf->{vga};
2775
2776     my $qxlnum = vga_conf_has_spice($vga);
2777     $vga = 'qxl' if $qxlnum;
2778
2779     if (!$vga) {
2780         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2781                                 $conf->{ostype} eq 'win7' ||
2782                                 $conf->{ostype} eq 'w2k8')) {
2783             $vga = 'std';
2784         } else {
2785             $vga = 'cirrus';
2786         }
2787     }
2788
2789     # enable absolute mouse coordinates (needed by vnc)
2790     my $tablet;
2791     if (defined($conf->{tablet})) {
2792         $tablet = $conf->{tablet};
2793     } else {
2794         $tablet = $defaults->{tablet};
2795         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2796         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2797     }
2798
2799     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2800
2801     my $kvm_off = 0;
2802     # host pci devices
2803     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2804         my $d = parse_hostpci($conf->{"hostpci$i"});
2805         next if !$d;
2806
2807         my $pcie = $d->{pcie};
2808         if($pcie){
2809             die "q35 machine model is not enabled" if !$q35;
2810             $pciaddr = print_pcie_addr("hostpci$i");
2811         }else{
2812             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2813         }
2814
2815         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2816         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2817         if ($xvga && $xvga ne '') {
2818             $kvm_off = 1;
2819             $vga = 'none';
2820             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2821                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2822             }
2823             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2824                 $xvga = "";
2825             }
2826         }
2827         my $pcidevices = $d->{pciid};
2828         my $multifunction = 1 if @$pcidevices > 1;
2829
2830         my $j=0;
2831         foreach my $pcidevice (@$pcidevices) {
2832
2833             my $id = "hostpci$i";
2834             $id .= ".$j" if $multifunction;
2835             my $addr = $pciaddr;
2836             $addr .= ".$j" if $multifunction;
2837             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2838
2839             if($j == 0){
2840                 $devicestr .= "$rombar$xvga";
2841                 $devicestr .= ",multifunction=on" if $multifunction;
2842             }
2843
2844             push @$devices, '-device', $devicestr;
2845             $j++;
2846         }
2847     }
2848
2849     # usb devices
2850     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2851         next if !$conf->{"usb$i"};
2852         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2853         next if !$d;
2854
2855         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2856         my $usbbus = '';
2857         if (defined($d->{usb3}) && $d->{usb3}) {
2858             $usbbus = ',bus=xhci.0';
2859         }
2860
2861         if (defined($d->{host})) {
2862             $d = parse_usb_device($d->{host});
2863             if (defined($d->{vendorid}) && defined($d->{productid})) {
2864                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2865             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2866                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2867             } elsif (defined($d->{spice}) && $d->{spice}) {
2868                 # usb redir support for spice, currently no usb3
2869                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2870                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2871             }
2872         }
2873     }
2874
2875     # serial devices
2876     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2877         if (my $path = $conf->{"serial$i"}) {
2878             if ($path eq 'socket') {
2879                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2880                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2881                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2882             } else {
2883                 die "no such serial device\n" if ! -c $path;
2884                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2885                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2886             }
2887         }
2888     }
2889
2890     # parallel devices
2891     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2892         if (my $path = $conf->{"parallel$i"}) {
2893             die "no such parallel device\n" if ! -c $path;
2894             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2895             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2896             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2897         }
2898     }
2899
2900     my $vmname = $conf->{name} || "vm$vmid";
2901
2902     push @$cmd, '-name', $vmname;
2903
2904     my $sockets = 1;
2905     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2906     $sockets = $conf->{sockets} if  $conf->{sockets};
2907
2908     my $cores = $conf->{cores} || 1;
2909
2910     my $maxcpus = $sockets * $cores;
2911
2912     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2913
2914     my $allowed_vcpus = $cpuinfo->{cpus};
2915
2916     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2917         if ($allowed_vcpus < $maxcpus);
2918
2919     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2920
2921     push @$cmd, '-nodefaults';
2922
2923     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2924
2925     my $bootindex_hash = {};
2926     my $i = 1;
2927     foreach my $o (split(//, $bootorder)) {
2928         $bootindex_hash->{$o} = $i*100;
2929         $i++;
2930     }
2931
2932     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2933
2934     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2935
2936     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2937
2938     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2939
2940     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2941         my $socket = vnc_socket($vmid);
2942         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2943     } else {
2944         push @$cmd, '-nographic';
2945     }
2946
2947     # time drift fix
2948     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2949
2950     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2951     my $useLocaltime = $conf->{localtime};
2952
2953     if ($ostype) {
2954         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2955
2956         if ($ostype =~ m/^w/) { # windows
2957             $useLocaltime = 1 if !defined($conf->{localtime});
2958
2959             # use time drift fix when acpi is enabled
2960             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2961                 $tdf = 1 if !defined($conf->{tdf});
2962             }
2963         }
2964
2965         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2966             $ostype eq 'wvista') {
2967             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2968             push @$cmd, '-no-hpet';
2969             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2970                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2971                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2972                 push @$cpuFlags , 'hv_time' if !$nokvm;
2973
2974             } else {
2975                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2976             }
2977         }
2978
2979         if ($ostype eq 'win7' || $ostype eq 'win8') {
2980             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2981         }
2982     }
2983
2984     push @$rtcFlags, 'driftfix=slew' if $tdf;
2985
2986     if ($nokvm) {
2987         push @$machineFlags, 'accel=tcg';
2988     } else {
2989         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2990     }
2991
2992     if ($machine_type) {
2993         push @$machineFlags, "type=${machine_type}";
2994     }
2995
2996     if ($conf->{startdate}) {
2997         push @$rtcFlags, "base=$conf->{startdate}";
2998     } elsif ($useLocaltime) {
2999         push @$rtcFlags, 'base=localtime';
3000     }
3001
3002     my $cpu = $nokvm ? "qemu64" : "kvm64";
3003     if (my $cputype = $conf->{cpu}) {
3004         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3005             or die "Cannot parse cpu description: $cputype\n";
3006         $cpu = $cpuconf->{cputype};
3007         $kvm_off = 1 if $cpuconf->{hidden};
3008     }
3009
3010     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3011
3012     push @$cpuFlags , '-x2apic'
3013         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3014
3015     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3016
3017     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3018
3019     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3020
3021         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3022         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3023     }
3024
3025     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3026
3027     push @$cpuFlags, 'kvm=off' if $kvm_off;
3028
3029     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3030
3031     push @$cmd, '-cpu', $cpu;
3032
3033     my $memory = $conf->{memory} || $defaults->{memory};
3034     my $static_memory = 0;
3035     my $dimm_memory = 0;
3036
3037     if ($hotplug_features->{memory}) {
3038         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3039         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3040         $static_memory = $STATICMEM;
3041         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3042         $dimm_memory = $memory - $static_memory;
3043         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3044
3045     } else {
3046
3047         $static_memory = $memory;
3048         push @$cmd, '-m', $static_memory;
3049     }
3050
3051     if ($conf->{numa}) {
3052
3053         my $numa_totalmemory = undef;
3054         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3055             next if !$conf->{"numa$i"};
3056             my $numa = parse_numa($conf->{"numa$i"});
3057             next if !$numa;
3058             # memory
3059             die "missing numa node$i memory value\n" if !$numa->{memory};
3060             my $numa_memory = $numa->{memory};
3061             $numa_totalmemory += $numa_memory;
3062             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3063
3064             # cpus
3065             my $cpus_start = $numa->{cpus}->{start};
3066             die "missing numa node$i cpus\n" if !defined($cpus_start);
3067             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3068             my $cpus = $cpus_start;
3069             if (defined($cpus_end)) {
3070                 $cpus .= "-$cpus_end";
3071                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3072             }
3073
3074             # hostnodes
3075             my $hostnodes_start = $numa->{hostnodes}->{start};
3076             if (defined($hostnodes_start)) {
3077                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3078                 my $hostnodes = $hostnodes_start;
3079                 if (defined($hostnodes_end)) {
3080                     $hostnodes .= "-$hostnodes_end";
3081                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3082                 }
3083
3084                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3085                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3086                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3087                 }
3088
3089                 # policy
3090                 my $policy = $numa->{policy};
3091                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3092                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3093             }
3094
3095             push @$cmd, '-object', $numa_object;
3096             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3097         }
3098
3099         die "total memory for NUMA nodes must be equal to vm static memory\n"
3100             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3101
3102         #if no custom tology, we split memory and cores across numa nodes
3103         if(!$numa_totalmemory) {
3104
3105             my $numa_memory = ($static_memory / $sockets) . "M";
3106
3107             for (my $i = 0; $i < $sockets; $i++)  {
3108
3109                 my $cpustart = ($cores * $i);
3110                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3111                 my $cpus = $cpustart;
3112                 $cpus .= "-$cpuend" if $cpuend;
3113
3114                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3115                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3116             }
3117         }
3118     }
3119
3120     if ($hotplug_features->{memory}) {
3121         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3122             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3123             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3124             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3125
3126             #if dimm_memory is not aligned to dimm map
3127             if($current_size > $memory) {
3128                  $conf->{memory} = $current_size;
3129                  PVE::QemuConfig->write_config($vmid, $conf);
3130             }
3131         });
3132     }
3133
3134     push @$cmd, '-S' if $conf->{freeze};
3135
3136     # set keyboard layout
3137     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3138     push @$cmd, '-k', $kb if $kb;
3139
3140     # enable sound
3141     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3142     #push @$cmd, '-soundhw', 'es1370';
3143     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3144
3145     if($conf->{agent}) {
3146         my $qgasocket = qmp_socket($vmid, 1);
3147         my $pciaddr = print_pci_addr("qga0", $bridges);
3148         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3149         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3150         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3151     }
3152
3153     my $spice_port;
3154
3155     if ($qxlnum) {
3156         if ($qxlnum > 1) {
3157             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3158                 for(my $i = 1; $i < $qxlnum; $i++){
3159                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3160                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3161                 }
3162             } else {
3163                 # assume other OS works like Linux
3164                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3165                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3166             }
3167         }
3168
3169         my $pciaddr = print_pci_addr("spice", $bridges);
3170
3171         my $nodename = PVE::INotify::nodename();
3172         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3173         $spice_port = PVE::Tools::next_spice_port($pfamily);
3174
3175         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3176
3177         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3178         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3179         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3180     }
3181
3182     # enable balloon by default, unless explicitly disabled
3183     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3184         $pciaddr = print_pci_addr("balloon0", $bridges);
3185         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3186     }
3187
3188     if ($conf->{watchdog}) {
3189         my $wdopts = parse_watchdog($conf->{watchdog});
3190         $pciaddr = print_pci_addr("watchdog", $bridges);
3191         my $watchdog = $wdopts->{model} || 'i6300esb';
3192         push @$devices, '-device', "$watchdog$pciaddr";
3193         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3194     }
3195
3196     my $vollist = [];
3197     my $scsicontroller = {};
3198     my $ahcicontroller = {};
3199     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3200
3201     # Add iscsi initiator name if available
3202     if (my $initiator = get_initiator_name()) {
3203         push @$devices, '-iscsi', "initiator-name=$initiator";
3204     }
3205
3206     foreach_drive($conf, sub {
3207         my ($ds, $drive) = @_;
3208
3209         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3210             push @$vollist, $drive->{file};
3211         }
3212
3213         $use_virtio = 1 if $ds =~ m/^virtio/;
3214
3215         if (drive_is_cdrom ($drive)) {
3216             if ($bootindex_hash->{d}) {
3217                 $drive->{bootindex} = $bootindex_hash->{d};
3218                 $bootindex_hash->{d} += 1;
3219             }
3220         } else {
3221             if ($bootindex_hash->{c}) {
3222                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3223                 $bootindex_hash->{c} += 1;
3224             }
3225         }
3226
3227         if($drive->{interface} eq 'virtio'){
3228            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3229         }
3230
3231         if ($drive->{interface} eq 'scsi') {
3232
3233             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3234
3235             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3236             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3237
3238             my $iothread = '';
3239             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3240                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3241                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3242             }
3243
3244             my $queues = '';
3245             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3246                 $queues = ",num_queues=$drive->{queues}";
3247             }
3248
3249             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3250             $scsicontroller->{$controller}=1;
3251         }
3252
3253         if ($drive->{interface} eq 'sata') {
3254            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3255            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3256            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3257            $ahcicontroller->{$controller}=1;
3258         }
3259
3260         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3261         push @$devices, '-drive',$drive_cmd;
3262         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3263     });
3264
3265     for (my $i = 0; $i < $MAX_NETS; $i++) {
3266          next if !$conf->{"net$i"};
3267          my $d = parse_net($conf->{"net$i"});
3268          next if !$d;
3269
3270          $use_virtio = 1 if $d->{model} eq 'virtio';
3271
3272          if ($bootindex_hash->{n}) {
3273             $d->{bootindex} = $bootindex_hash->{n};
3274             $bootindex_hash->{n} += 1;
3275          }
3276
3277          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3278          push @$devices, '-netdev', $netdevfull;
3279
3280          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3281          push @$devices, '-device', $netdevicefull;
3282     }
3283
3284     if (!$q35) {
3285         # add pci bridges
3286         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3287            $bridges->{1} = 1;
3288            $bridges->{2} = 1;
3289         }
3290
3291         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3292
3293         while (my ($k, $v) = each %$bridges) {
3294             $pciaddr = print_pci_addr("pci.$k");
3295             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3296         }
3297     }
3298
3299     # add custom args
3300     if ($conf->{args}) {
3301         my $aa = PVE::Tools::split_args($conf->{args});
3302         push @$cmd, @$aa;
3303     }
3304
3305     push @$cmd, @$devices;
3306     push @$cmd, '-rtc', join(',', @$rtcFlags)
3307         if scalar(@$rtcFlags);
3308     push @$cmd, '-machine', join(',', @$machineFlags)
3309         if scalar(@$machineFlags);
3310     push @$cmd, '-global', join(',', @$globalFlags)
3311         if scalar(@$globalFlags);
3312
3313     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3314 }
3315
3316 sub vnc_socket {
3317     my ($vmid) = @_;
3318     return "${var_run_tmpdir}/$vmid.vnc";
3319 }
3320
3321 sub spice_port {
3322     my ($vmid) = @_;
3323
3324     my $res = vm_mon_cmd($vmid, 'query-spice');
3325
3326     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3327 }
3328
3329 sub qmp_socket {
3330     my ($vmid, $qga) = @_;
3331     my $sockettype = $qga ? 'qga' : 'qmp';
3332     return "${var_run_tmpdir}/$vmid.$sockettype";
3333 }
3334
3335 sub pidfile_name {
3336     my ($vmid) = @_;
3337     return "${var_run_tmpdir}/$vmid.pid";
3338 }
3339
3340 sub vm_devices_list {
3341     my ($vmid) = @_;
3342
3343     my $res = vm_mon_cmd($vmid, 'query-pci');
3344     my $devices = {};
3345     foreach my $pcibus (@$res) {
3346         foreach my $device (@{$pcibus->{devices}}) {
3347             next if !$device->{'qdev_id'};
3348             if ($device->{'pci_bridge'}) {
3349                 $devices->{$device->{'qdev_id'}} = 1;
3350                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3351                     next if !$bridge_device->{'qdev_id'};
3352                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3353                     $devices->{$device->{'qdev_id'}}++;
3354                 }
3355             } else {
3356                 $devices->{$device->{'qdev_id'}} = 1;
3357             }
3358         }
3359     }
3360
3361     my $resblock = vm_mon_cmd($vmid, 'query-block');
3362     foreach my $block (@$resblock) {
3363         if($block->{device} =~ m/^drive-(\S+)/){
3364                 $devices->{$1} = 1;
3365         }
3366     }
3367
3368     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3369     foreach my $mice (@$resmice) {
3370         if ($mice->{name} eq 'QEMU HID Tablet') {
3371             $devices->{tablet} = 1;
3372             last;
3373         }
3374     }
3375
3376     return $devices;
3377 }
3378
3379 sub vm_deviceplug {
3380     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3381
3382     my $q35 = machine_type_is_q35($conf);
3383
3384     my $devices_list = vm_devices_list($vmid);
3385     return 1 if defined($devices_list->{$deviceid});
3386
3387     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3388
3389     if ($deviceid eq 'tablet') {
3390
3391         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3392
3393     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3394
3395         qemu_iothread_add($vmid, $deviceid, $device);
3396
3397         qemu_driveadd($storecfg, $vmid, $device);
3398         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3399
3400         qemu_deviceadd($vmid, $devicefull);
3401         eval { qemu_deviceaddverify($vmid, $deviceid); };
3402         if (my $err = $@) {
3403             eval { qemu_drivedel($vmid, $deviceid); };
3404             warn $@ if $@;
3405             die $err;
3406         }
3407
3408     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3409
3410
3411         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3412         my $pciaddr = print_pci_addr($deviceid);
3413         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3414
3415         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3416
3417         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3418             qemu_iothread_add($vmid, $deviceid, $device);
3419             $devicefull .= ",iothread=iothread-$deviceid";
3420         }
3421
3422         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3423             $devicefull .= ",num_queues=$device->{queues}";
3424         }
3425
3426         qemu_deviceadd($vmid, $devicefull);
3427         qemu_deviceaddverify($vmid, $deviceid);
3428
3429     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3430
3431         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3432         qemu_driveadd($storecfg, $vmid, $device);
3433
3434         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3435         eval { qemu_deviceadd($vmid, $devicefull); };
3436         if (my $err = $@) {
3437             eval { qemu_drivedel($vmid, $deviceid); };
3438             warn $@ if $@;
3439             die $err;
3440         }
3441
3442     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3443
3444         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3445
3446         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3447         my $use_old_bios_files = undef;
3448         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3449
3450         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3451         qemu_deviceadd($vmid, $netdevicefull);
3452         eval { qemu_deviceaddverify($vmid, $deviceid); };
3453         if (my $err = $@) {
3454             eval { qemu_netdevdel($vmid, $deviceid); };
3455             warn $@ if $@;
3456             die $err;
3457         }
3458
3459     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3460
3461         my $bridgeid = $2;
3462         my $pciaddr = print_pci_addr($deviceid);
3463         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3464
3465         qemu_deviceadd($vmid, $devicefull);
3466         qemu_deviceaddverify($vmid, $deviceid);
3467
3468     } else {
3469         die "can't hotplug device '$deviceid'\n";
3470     }
3471
3472     return 1;
3473 }
3474
3475 # fixme: this should raise exceptions on error!
3476 sub vm_deviceunplug {
3477     my ($vmid, $conf, $deviceid) = @_;
3478
3479     my $devices_list = vm_devices_list($vmid);
3480     return 1 if !defined($devices_list->{$deviceid});
3481
3482     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3483
3484     if ($deviceid eq 'tablet') {
3485
3486         qemu_devicedel($vmid, $deviceid);
3487
3488     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3489
3490         qemu_devicedel($vmid, $deviceid);
3491         qemu_devicedelverify($vmid, $deviceid);
3492         qemu_drivedel($vmid, $deviceid);
3493         qemu_iothread_del($conf, $vmid, $deviceid);
3494
3495     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3496
3497         qemu_devicedel($vmid, $deviceid);
3498         qemu_devicedelverify($vmid, $deviceid);
3499         qemu_iothread_del($conf, $vmid, $deviceid);
3500
3501     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3502
3503         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3504         my $device = parse_drive($deviceid, $conf->{$deviceid});
3505         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3506
3507         qemu_devicedel($vmid, $deviceid);
3508         qemu_drivedel($vmid, $deviceid);
3509         qemu_deletescsihw($conf, $vmid, $deviceid);
3510
3511     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3512
3513         qemu_devicedel($vmid, $deviceid);
3514         qemu_devicedelverify($vmid, $deviceid);
3515         qemu_netdevdel($vmid, $deviceid);
3516
3517     } else {
3518         die "can't unplug device '$deviceid'\n";
3519     }
3520
3521     return 1;
3522 }
3523
3524 sub qemu_deviceadd {
3525     my ($vmid, $devicefull) = @_;
3526
3527     $devicefull = "driver=".$devicefull;
3528     my %options =  split(/[=,]/, $devicefull);
3529
3530     vm_mon_cmd($vmid, "device_add" , %options);
3531 }
3532
3533 sub qemu_devicedel {
3534     my ($vmid, $deviceid) = @_;
3535
3536     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3537 }
3538
3539 sub qemu_iothread_add {
3540     my($vmid, $deviceid, $device) = @_;
3541
3542     if ($device->{iothread}) {
3543         my $iothreads = vm_iothreads_list($vmid);
3544         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3545     }
3546 }
3547
3548 sub qemu_iothread_del {
3549     my($conf, $vmid, $deviceid) = @_;
3550
3551     my $device = parse_drive($deviceid, $conf->{$deviceid});
3552     if ($device->{iothread}) {
3553         my $iothreads = vm_iothreads_list($vmid);
3554         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3555     }
3556 }
3557
3558 sub qemu_objectadd {
3559     my($vmid, $objectid, $qomtype) = @_;
3560
3561     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3562
3563     return 1;
3564 }
3565
3566 sub qemu_objectdel {
3567     my($vmid, $objectid) = @_;
3568
3569     vm_mon_cmd($vmid, "object-del", id => $objectid);
3570
3571     return 1;
3572 }
3573
3574 sub qemu_driveadd {
3575     my ($storecfg, $vmid, $device) = @_;
3576
3577     my $drive = print_drive_full($storecfg, $vmid, $device);
3578     $drive =~ s/\\/\\\\/g;
3579     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3580
3581     # If the command succeeds qemu prints: "OK"
3582     return 1 if $ret =~ m/OK/s;
3583
3584     die "adding drive failed: $ret\n";
3585 }
3586
3587 sub qemu_drivedel {
3588     my($vmid, $deviceid) = @_;
3589
3590     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3591     $ret =~ s/^\s+//;
3592
3593     return 1 if $ret eq "";
3594
3595     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3596     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3597
3598     die "deleting drive $deviceid failed : $ret\n";
3599 }
3600
3601 sub qemu_deviceaddverify {
3602     my ($vmid, $deviceid) = @_;
3603
3604     for (my $i = 0; $i <= 5; $i++) {
3605          my $devices_list = vm_devices_list($vmid);
3606          return 1 if defined($devices_list->{$deviceid});
3607          sleep 1;
3608     }
3609
3610     die "error on hotplug device '$deviceid'\n";
3611 }
3612
3613
3614 sub qemu_devicedelverify {
3615     my ($vmid, $deviceid) = @_;
3616
3617     # need to verify that the device is correctly removed as device_del
3618     # is async and empty return is not reliable
3619
3620     for (my $i = 0; $i <= 5; $i++) {
3621          my $devices_list = vm_devices_list($vmid);
3622          return 1 if !defined($devices_list->{$deviceid});
3623          sleep 1;
3624     }
3625
3626     die "error on hot-unplugging device '$deviceid'\n";
3627 }
3628
3629 sub qemu_findorcreatescsihw {
3630     my ($storecfg, $conf, $vmid, $device) = @_;
3631
3632     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3633
3634     my $scsihwid="$controller_prefix$controller";
3635     my $devices_list = vm_devices_list($vmid);
3636
3637     if(!defined($devices_list->{$scsihwid})) {
3638         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3639     }
3640
3641     return 1;
3642 }
3643
3644 sub qemu_deletescsihw {
3645     my ($conf, $vmid, $opt) = @_;
3646
3647     my $device = parse_drive($opt, $conf->{$opt});
3648
3649     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3650         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3651         return 1;
3652     }
3653
3654     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3655
3656     my $devices_list = vm_devices_list($vmid);
3657     foreach my $opt (keys %{$devices_list}) {
3658         if (PVE::QemuServer::is_valid_drivename($opt)) {
3659             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3660             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3661                 return 1;
3662             }
3663         }
3664     }
3665
3666     my $scsihwid="scsihw$controller";
3667
3668     vm_deviceunplug($vmid, $conf, $scsihwid);
3669
3670     return 1;
3671 }
3672
3673 sub qemu_add_pci_bridge {
3674     my ($storecfg, $conf, $vmid, $device) = @_;
3675
3676     my $bridges = {};
3677
3678     my $bridgeid;
3679
3680     print_pci_addr($device, $bridges);
3681
3682     while (my ($k, $v) = each %$bridges) {
3683         $bridgeid = $k;
3684     }
3685     return 1 if !defined($bridgeid) || $bridgeid < 1;
3686
3687     my $bridge = "pci.$bridgeid";
3688     my $devices_list = vm_devices_list($vmid);
3689
3690     if (!defined($devices_list->{$bridge})) {
3691         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3692     }
3693
3694     return 1;
3695 }
3696
3697 sub qemu_set_link_status {
3698     my ($vmid, $device, $up) = @_;
3699
3700     vm_mon_cmd($vmid, "set_link", name => $device,
3701                up => $up ? JSON::true : JSON::false);
3702 }
3703
3704 sub qemu_netdevadd {
3705     my ($vmid, $conf, $device, $deviceid) = @_;
3706
3707     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3708     my %options =  split(/[=,]/, $netdev);
3709
3710     vm_mon_cmd($vmid, "netdev_add",  %options);
3711     return 1;
3712 }
3713
3714 sub qemu_netdevdel {
3715     my ($vmid, $deviceid) = @_;
3716
3717     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3718 }
3719
3720 sub qemu_cpu_hotplug {
3721     my ($vmid, $conf, $vcpus) = @_;
3722
3723     my $sockets = 1;
3724     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3725     $sockets = $conf->{sockets} if  $conf->{sockets};
3726     my $cores = $conf->{cores} || 1;
3727     my $maxcpus = $sockets * $cores;
3728
3729     $vcpus = $maxcpus if !$vcpus;
3730
3731     die "you can't add more vcpus than maxcpus\n"
3732         if $vcpus > $maxcpus;
3733
3734     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3735     die "online cpu unplug is not yet possible\n"
3736         if $vcpus < $currentvcpus;
3737
3738     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3739     die "vcpus in running vm is different than configuration\n"
3740         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3741
3742     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3743         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3744     }
3745 }
3746
3747 sub qemu_memory_hotplug {
3748     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3749
3750     return $value if !check_running($vmid);
3751
3752     my $memory = $conf->{memory} || $defaults->{memory};
3753     $value = $defaults->{memory} if !$value;
3754     return $value if $value == $memory;
3755
3756     my $static_memory = $STATICMEM;
3757     my $dimm_memory = $memory - $static_memory;
3758
3759     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3760     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3761
3762
3763     my $sockets = 1;
3764     $sockets = $conf->{sockets} if $conf->{sockets};
3765
3766     if($value > $memory) {
3767
3768         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3769             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3770
3771                 return if $current_size <= $conf->{memory};
3772
3773                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3774                 if (my $err = $@) {
3775                     eval { qemu_objectdel($vmid, "mem-$name"); };
3776                     die $err;
3777                 }
3778
3779                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3780                 if (my $err = $@) {
3781                     eval { qemu_objectdel($vmid, "mem-$name"); };
3782                     die $err;
3783                 }
3784                 #update conf after each succesful module hotplug
3785                 $conf->{memory} = $current_size;
3786                 PVE::QemuConfig->write_config($vmid, $conf);
3787         });
3788
3789     } else {
3790
3791         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3792             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3793
3794                 return if $current_size >= $conf->{memory};
3795                 print "try to unplug memory dimm $name\n";
3796
3797                 my $retry = 0;
3798                 while (1) {
3799                     eval { qemu_devicedel($vmid, $name) };
3800                     sleep 3;
3801                     my $dimm_list = qemu_dimm_list($vmid);
3802                     last if !$dimm_list->{$name};
3803                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3804                     $retry++;
3805                 }
3806
3807                 #update conf after each succesful module unplug
3808                 $conf->{memory} = $current_size;
3809
3810                 eval { qemu_objectdel($vmid, "mem-$name"); };
3811                 PVE::QemuConfig->write_config($vmid, $conf);
3812         });
3813     }
3814 }
3815
3816 sub qemu_dimm_list {
3817     my ($vmid) = @_;
3818
3819     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3820     my $dimms = {};
3821
3822     foreach my $dimm (@$dimmarray) {
3823
3824         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3825         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3826         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3827         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3828         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3829     }
3830     return $dimms;
3831 }
3832
3833 sub qemu_block_set_io_throttle {
3834     my ($vmid, $deviceid,
3835         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3836         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3837
3838     return if !check_running($vmid) ;
3839
3840     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3841         bps => int($bps),
3842         bps_rd => int($bps_rd),
3843         bps_wr => int($bps_wr),
3844         iops => int($iops),
3845         iops_rd => int($iops_rd),
3846         iops_wr => int($iops_wr),
3847         bps_max => int($bps_max),
3848         bps_rd_max => int($bps_rd_max),
3849         bps_wr_max => int($bps_wr_max),
3850         iops_max => int($iops_max),
3851         iops_rd_max => int($iops_rd_max),
3852         iops_wr_max => int($iops_wr_max)
3853     );
3854
3855 }
3856
3857 # old code, only used to shutdown old VM after update
3858 sub __read_avail {
3859     my ($fh, $timeout) = @_;
3860
3861     my $sel = new IO::Select;
3862     $sel->add($fh);
3863
3864     my $res = '';
3865     my $buf;
3866
3867     my @ready;
3868     while (scalar (@ready = $sel->can_read($timeout))) {
3869         my $count;
3870         if ($count = $fh->sysread($buf, 8192)) {
3871             if ($buf =~ /^(.*)\(qemu\) $/s) {
3872                 $res .= $1;
3873                 last;
3874             } else {
3875                 $res .= $buf;
3876             }
3877         } else {
3878             if (!defined($count)) {
3879                 die "$!\n";
3880             }
3881             last;
3882         }
3883     }
3884
3885     die "monitor read timeout\n" if !scalar(@ready);
3886
3887     return $res;
3888 }
3889
3890 # old code, only used to shutdown old VM after update
3891 sub vm_monitor_command {
3892     my ($vmid, $cmdstr, $nocheck) = @_;
3893
3894     my $res;
3895
3896     eval {
3897         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3898
3899         my $sname = "${var_run_tmpdir}/$vmid.mon";
3900
3901         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3902             die "unable to connect to VM $vmid socket - $!\n";
3903
3904         my $timeout = 3;
3905
3906         # hack: migrate sometime blocks the monitor (when migrate_downtime
3907         # is set)
3908         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3909             $timeout = 60*60; # 1 hour
3910         }
3911
3912         # read banner;
3913         my $data = __read_avail($sock, $timeout);
3914
3915         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3916             die "got unexpected qemu monitor banner\n";
3917         }
3918
3919         my $sel = new IO::Select;
3920         $sel->add($sock);
3921
3922         if (!scalar(my @ready = $sel->can_write($timeout))) {
3923             die "monitor write error - timeout";
3924         }
3925
3926         my $fullcmd = "$cmdstr\r";
3927
3928         # syslog('info', "VM $vmid monitor command: $cmdstr");
3929
3930         my $b;
3931         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3932             die "monitor write error - $!";
3933         }
3934
3935         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3936
3937         $timeout = 20;
3938
3939         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3940             $timeout = 60*60; # 1 hour
3941         } elsif ($cmdstr =~ m/^(eject|change)/) {
3942             $timeout = 60; # note: cdrom mount command is slow
3943         }
3944         if ($res = __read_avail($sock, $timeout)) {
3945
3946             my @lines = split("\r?\n", $res);
3947
3948             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3949
3950             $res = join("\n", @lines);
3951             $res .= "\n";
3952         }
3953     };
3954
3955     my $err = $@;
3956
3957     if ($err) {
3958         syslog("err", "VM $vmid monitor command failed - $err");
3959         die $err;
3960     }
3961
3962     return $res;
3963 }
3964
3965 sub qemu_block_resize {
3966     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3967
3968     my $running = check_running($vmid);
3969
3970     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3971
3972     return if !$running;
3973
3974     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3975
3976 }
3977
3978 sub qemu_volume_snapshot {
3979     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3980
3981     my $running = check_running($vmid);
3982
3983     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3984         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3985     } else {
3986         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3987     }
3988 }
3989
3990 sub qemu_volume_snapshot_delete {
3991     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3992
3993     my $running = check_running($vmid);
3994
3995     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3996
3997     return if !$running;
3998
3999     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4000 }
4001
4002 sub set_migration_caps {
4003     my ($vmid) = @_;
4004
4005     my $cap_ref = [];
4006
4007     my $enabled_cap = {
4008         "auto-converge" => 1,
4009         "xbzrle" => 1,
4010         "x-rdma-pin-all" => 0,
4011         "zero-blocks" => 0,
4012         "compress" => 0
4013     };
4014
4015     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4016
4017     for my $supported_capability (@$supported_capabilities) {
4018         push @$cap_ref, {
4019             capability => $supported_capability->{capability},
4020             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4021         };
4022     }
4023
4024     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4025 }
4026
4027 my $fast_plug_option = {
4028     'lock' => 1,
4029     'name' => 1,
4030     'onboot' => 1,
4031     'shares' => 1,
4032     'startup' => 1,
4033     'description' => 1,
4034 };
4035
4036 # hotplug changes in [PENDING]
4037 # $selection hash can be used to only apply specified options, for
4038 # example: { cores => 1 } (only apply changed 'cores')
4039 # $errors ref is used to return error messages
4040 sub vmconfig_hotplug_pending {
4041     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4042
4043     my $defaults = load_defaults();
4044
4045     # commit values which do not have any impact on running VM first
4046     # Note: those option cannot raise errors, we we do not care about
4047     # $selection and always apply them.
4048
4049     my $add_error = sub {
4050         my ($opt, $msg) = @_;
4051         $errors->{$opt} = "hotplug problem - $msg";
4052     };
4053
4054     my $changes = 0;
4055     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4056         if ($fast_plug_option->{$opt}) {
4057             $conf->{$opt} = $conf->{pending}->{$opt};
4058             delete $conf->{pending}->{$opt};
4059             $changes = 1;
4060         }
4061     }
4062
4063     if ($changes) {
4064         PVE::QemuConfig->write_config($vmid, $conf);
4065         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4066     }
4067
4068     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4069
4070     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4071     while (my ($opt, $force) = each %$pending_delete_hash) {
4072         next if $selection && !$selection->{$opt};
4073         eval {
4074             if ($opt eq 'hotplug') {
4075                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4076             } elsif ($opt eq 'tablet') {
4077                 die "skip\n" if !$hotplug_features->{usb};
4078                 if ($defaults->{tablet}) {
4079                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4080                 } else {
4081                     vm_deviceunplug($vmid, $conf, $opt);
4082                 }
4083             } elsif ($opt eq 'vcpus') {
4084                 die "skip\n" if !$hotplug_features->{cpu};
4085                 qemu_cpu_hotplug($vmid, $conf, undef);
4086             } elsif ($opt eq 'balloon') {
4087                 # enable balloon device is not hotpluggable
4088                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4089             } elsif ($fast_plug_option->{$opt}) {
4090                 # do nothing
4091             } elsif ($opt =~ m/^net(\d+)$/) {
4092                 die "skip\n" if !$hotplug_features->{network};
4093                 vm_deviceunplug($vmid, $conf, $opt);
4094             } elsif (is_valid_drivename($opt)) {
4095                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4096                 vm_deviceunplug($vmid, $conf, $opt);
4097                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4098             } elsif ($opt =~ m/^memory$/) {
4099                 die "skip\n" if !$hotplug_features->{memory};
4100                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4101             } elsif ($opt eq 'cpuunits') {
4102                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4103             } elsif ($opt eq 'cpulimit') {
4104                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4105             } else {
4106                 die "skip\n";
4107             }
4108         };
4109         if (my $err = $@) {
4110             &$add_error($opt, $err) if $err ne "skip\n";
4111         } else {
4112             # save new config if hotplug was successful
4113             delete $conf->{$opt};
4114             vmconfig_undelete_pending_option($conf, $opt);
4115             PVE::QemuConfig->write_config($vmid, $conf);
4116             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4117         }
4118     }
4119
4120     foreach my $opt (keys %{$conf->{pending}}) {
4121         next if $selection && !$selection->{$opt};
4122         my $value = $conf->{pending}->{$opt};
4123         eval {
4124             if ($opt eq 'hotplug') {
4125                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4126             } elsif ($opt eq 'tablet') {
4127                 die "skip\n" if !$hotplug_features->{usb};
4128                 if ($value == 1) {
4129                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4130                 } elsif ($value == 0) {
4131                     vm_deviceunplug($vmid, $conf, $opt);
4132                 }
4133             } elsif ($opt eq 'vcpus') {
4134                 die "skip\n" if !$hotplug_features->{cpu};
4135                 qemu_cpu_hotplug($vmid, $conf, $value);
4136             } elsif ($opt eq 'balloon') {
4137                 # enable/disable balloning device is not hotpluggable
4138                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4139                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4140                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4141
4142                 # allow manual ballooning if shares is set to zero
4143                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4144                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4145                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4146                 }
4147             } elsif ($opt =~ m/^net(\d+)$/) {
4148                 # some changes can be done without hotplug
4149                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4150                                     $vmid, $opt, $value);
4151             } elsif (is_valid_drivename($opt)) {
4152                 # some changes can be done without hotplug
4153                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4154                                      $vmid, $opt, $value, 1);
4155             } elsif ($opt =~ m/^memory$/) { #dimms
4156                 die "skip\n" if !$hotplug_features->{memory};
4157                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4158             } elsif ($opt eq 'cpuunits') {
4159                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4160             } elsif ($opt eq 'cpulimit') {
4161                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4162                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4163             } else {
4164                 die "skip\n";  # skip non-hot-pluggable options
4165             }
4166         };
4167         if (my $err = $@) {
4168             &$add_error($opt, $err) if $err ne "skip\n";
4169         } else {
4170             # save new config if hotplug was successful
4171             $conf->{$opt} = $value;
4172             delete $conf->{pending}->{$opt};
4173             PVE::QemuConfig->write_config($vmid, $conf);
4174             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4175         }
4176     }
4177 }
4178
4179 sub try_deallocate_drive {
4180     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4181
4182     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4183         my $volid = $drive->{file};
4184         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4185             my $sid = PVE::Storage::parse_volume_id($volid);
4186             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4187
4188             # check if the disk is really unused
4189             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4190                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4191             PVE::Storage::vdisk_free($storecfg, $volid);
4192             return 1;
4193         } else {
4194             # If vm is not owner of this disk remove from config
4195             return 1;
4196         }
4197     }
4198
4199     return undef;
4200 }
4201
4202 sub vmconfig_delete_or_detach_drive {
4203     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4204
4205     my $drive = parse_drive($opt, $conf->{$opt});
4206
4207     my $rpcenv = PVE::RPCEnvironment::get();
4208     my $authuser = $rpcenv->get_user();
4209
4210     if ($force) {
4211         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4212         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4213     } else {
4214         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4215     }
4216 }
4217
4218 sub vmconfig_apply_pending {
4219     my ($vmid, $conf, $storecfg) = @_;
4220
4221     # cold plug
4222
4223     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4224     while (my ($opt, $force) = each %$pending_delete_hash) {
4225         die "internal error" if $opt =~ m/^unused/;
4226         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4227         if (!defined($conf->{$opt})) {
4228             vmconfig_undelete_pending_option($conf, $opt);
4229             PVE::QemuConfig->write_config($vmid, $conf);
4230         } elsif (is_valid_drivename($opt)) {
4231             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4232             vmconfig_undelete_pending_option($conf, $opt);
4233             delete $conf->{$opt};
4234             PVE::QemuConfig->write_config($vmid, $conf);
4235         } else {
4236             vmconfig_undelete_pending_option($conf, $opt);
4237             delete $conf->{$opt};
4238             PVE::QemuConfig->write_config($vmid, $conf);
4239         }
4240     }
4241
4242     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4243
4244     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4245         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4246
4247         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4248             # skip if nothing changed
4249         } elsif (is_valid_drivename($opt)) {
4250             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4251                 if defined($conf->{$opt});
4252             $conf->{$opt} = $conf->{pending}->{$opt};
4253         } else {
4254             $conf->{$opt} = $conf->{pending}->{$opt};
4255         }
4256
4257         delete $conf->{pending}->{$opt};
4258         PVE::QemuConfig->write_config($vmid, $conf);
4259     }
4260 }
4261
4262 my $safe_num_ne = sub {
4263     my ($a, $b) = @_;
4264
4265     return 0 if !defined($a) && !defined($b);
4266     return 1 if !defined($a);
4267     return 1 if !defined($b);
4268
4269     return $a != $b;
4270 };
4271
4272 my $safe_string_ne = sub {
4273     my ($a, $b) = @_;
4274
4275     return 0 if !defined($a) && !defined($b);
4276     return 1 if !defined($a);
4277     return 1 if !defined($b);
4278
4279     return $a ne $b;
4280 };
4281
4282 sub vmconfig_update_net {
4283     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4284
4285     my $newnet = parse_net($value);
4286
4287     if ($conf->{$opt}) {
4288         my $oldnet = parse_net($conf->{$opt});
4289
4290         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4291             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4292             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4293             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4294
4295             # for non online change, we try to hot-unplug
4296             die "skip\n" if !$hotplug;
4297             vm_deviceunplug($vmid, $conf, $opt);
4298         } else {
4299
4300             die "internal error" if $opt !~ m/net(\d+)/;
4301             my $iface = "tap${vmid}i$1";
4302
4303             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4304                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4305                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4306                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4307                 PVE::Network::tap_unplug($iface);
4308                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4309             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4310                 # Rate can be applied on its own but any change above needs to
4311                 # include the rate in tap_plug since OVS resets everything.
4312                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4313             }
4314
4315             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4316                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4317             }
4318
4319             return 1;
4320         }
4321     }
4322
4323     if ($hotplug) {
4324         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4325     } else {
4326         die "skip\n";
4327     }
4328 }
4329
4330 sub vmconfig_update_disk {
4331     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4332
4333     # fixme: do we need force?
4334
4335     my $drive = parse_drive($opt, $value);
4336
4337     if ($conf->{$opt}) {
4338
4339         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4340
4341             my $media = $drive->{media} || 'disk';
4342             my $oldmedia = $old_drive->{media} || 'disk';
4343             die "unable to change media type\n" if $media ne $oldmedia;
4344
4345             if (!drive_is_cdrom($old_drive)) {
4346
4347                 if ($drive->{file} ne $old_drive->{file}) {
4348
4349                     die "skip\n" if !$hotplug;
4350
4351                     # unplug and register as unused
4352                     vm_deviceunplug($vmid, $conf, $opt);
4353                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4354
4355                 } else {
4356                     # update existing disk
4357
4358                     # skip non hotpluggable value
4359                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4360                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4361                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4362                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4363                         die "skip\n";
4364                     }
4365
4366                     # apply throttle
4367                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4368                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4369                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4370                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4371                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4372                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4373                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4374                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4375                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4376                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4377                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4378                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4379
4380                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4381                                                    ($drive->{mbps} || 0)*1024*1024,
4382                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4383                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4384                                                    $drive->{iops} || 0,
4385                                                    $drive->{iops_rd} || 0,
4386                                                    $drive->{iops_wr} || 0,
4387                                                    ($drive->{mbps_max} || 0)*1024*1024,
4388                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4389                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4390                                                    $drive->{iops_max} || 0,
4391                                                    $drive->{iops_rd_max} || 0,
4392                                                    $drive->{iops_wr_max} || 0);
4393
4394                     }
4395
4396                     return 1;
4397                 }
4398
4399             } else { # cdrom
4400
4401                 if ($drive->{file} eq 'none') {
4402                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4403                 } else {
4404                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4405                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4406                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4407                 }
4408
4409                 return 1;
4410             }
4411         }
4412     }
4413
4414     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4415     # hotplug new disks
4416     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4417     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4418 }
4419
4420 sub vm_start {
4421     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4422         $forcemachine, $spice_ticket) = @_;
4423
4424     PVE::QemuConfig->lock_config($vmid, sub {
4425         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4426
4427         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4428
4429         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4430
4431         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4432
4433         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4434             vmconfig_apply_pending($vmid, $conf, $storecfg);
4435             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4436         }
4437
4438         my $defaults = load_defaults();
4439
4440         # set environment variable useful inside network script
4441         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4442
4443         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4444
4445         my $migrate_port = 0;
4446         my $migrate_uri;
4447         if ($statefile) {
4448             if ($statefile eq 'tcp') {
4449                 my $localip = "localhost";
4450                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4451                 my $nodename = PVE::INotify::nodename();
4452                 if ($datacenterconf->{migration_unsecure}) {
4453                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4454                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4455                 }
4456                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4457                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4458                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4459                 push @$cmd, '-incoming', $migrate_uri;
4460                 push @$cmd, '-S';
4461             } else {
4462                 push @$cmd, '-loadstate', $statefile;
4463             }
4464         } elsif ($paused) {
4465             push @$cmd, '-S';
4466         }
4467
4468         # host pci devices
4469         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4470           my $d = parse_hostpci($conf->{"hostpci$i"});
4471           next if !$d;
4472           my $pcidevices = $d->{pciid};
4473           foreach my $pcidevice (@$pcidevices) {
4474                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4475
4476                 my $info = pci_device_info("0000:$pciid");
4477                 die "IOMMU not present\n" if !check_iommu_support();
4478                 die "no pci device info for device '$pciid'\n" if !$info;
4479                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4480                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4481           }
4482         }
4483
4484         PVE::Storage::activate_volumes($storecfg, $vollist);
4485
4486         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4487                     umask => 0077); };
4488
4489         if (my $err = $@) {
4490             # deactivate volumes if start fails
4491             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4492             die "start failed: $err";
4493         }
4494
4495         print "migration listens on $migrate_uri\n" if $migrate_uri;
4496
4497         if ($statefile && $statefile ne 'tcp')  {
4498             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4499             warn $@ if $@;
4500         }
4501
4502         if ($migratedfrom) {
4503
4504             eval {
4505                 set_migration_caps($vmid);
4506             };
4507             warn $@ if $@;
4508
4509             if ($spice_port) {
4510                 print "spice listens on port $spice_port\n";
4511                 if ($spice_ticket) {
4512                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4513                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4514                 }
4515             }
4516
4517         } else {
4518
4519             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4520                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4521                     if $conf->{balloon};
4522             }
4523
4524             foreach my $opt (keys %$conf) {
4525                 next if $opt !~  m/^net\d+$/;
4526                 my $nicconf = parse_net($conf->{$opt});
4527                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4528             }
4529         }
4530
4531         vm_mon_cmd_nocheck($vmid, 'qom-set',
4532                     path => "machine/peripheral/balloon0",
4533                     property => "guest-stats-polling-interval",
4534                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4535
4536     });
4537 }
4538
4539 sub vm_mon_cmd {
4540     my ($vmid, $execute, %params) = @_;
4541
4542     my $cmd = { execute => $execute, arguments => \%params };
4543     vm_qmp_command($vmid, $cmd);
4544 }
4545
4546 sub vm_mon_cmd_nocheck {
4547     my ($vmid, $execute, %params) = @_;
4548
4549     my $cmd = { execute => $execute, arguments => \%params };
4550     vm_qmp_command($vmid, $cmd, 1);
4551 }
4552
4553 sub vm_qmp_command {
4554     my ($vmid, $cmd, $nocheck) = @_;
4555
4556     my $res;
4557
4558     my $timeout;
4559     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4560         $timeout = $cmd->{arguments}->{timeout};
4561         delete $cmd->{arguments}->{timeout};
4562     }
4563
4564     eval {
4565         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4566         my $sname = qmp_socket($vmid);
4567         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4568             my $qmpclient = PVE::QMPClient->new();
4569
4570             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4571         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4572             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4573                 if scalar(%{$cmd->{arguments}});
4574             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4575         } else {
4576             die "unable to open monitor socket\n";
4577         }
4578     };
4579     if (my $err = $@) {
4580         syslog("err", "VM $vmid qmp command failed - $err");
4581         die $err;
4582     }
4583
4584     return $res;
4585 }
4586
4587 sub vm_human_monitor_command {
4588     my ($vmid, $cmdline) = @_;
4589
4590     my $res;
4591
4592     my $cmd = {
4593         execute => 'human-monitor-command',
4594         arguments => { 'command-line' => $cmdline},
4595     };
4596
4597     return vm_qmp_command($vmid, $cmd);
4598 }
4599
4600 sub vm_commandline {
4601     my ($storecfg, $vmid) = @_;
4602
4603     my $conf = PVE::QemuConfig->load_config($vmid);
4604
4605     my $defaults = load_defaults();
4606
4607     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4608
4609     return join(' ', @$cmd);
4610 }
4611
4612 sub vm_reset {
4613     my ($vmid, $skiplock) = @_;
4614
4615     PVE::QemuConfig->lock_config($vmid, sub {
4616
4617         my $conf = PVE::QemuConfig->load_config($vmid);
4618
4619         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4620
4621         vm_mon_cmd($vmid, "system_reset");
4622     });
4623 }
4624
4625 sub get_vm_volumes {
4626     my ($conf) = @_;
4627
4628     my $vollist = [];
4629     foreach_volid($conf, sub {
4630         my ($volid, $is_cdrom) = @_;
4631
4632         return if $volid =~ m|^/|;
4633
4634         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4635         return if !$sid;
4636
4637         push @$vollist, $volid;
4638     });
4639
4640     return $vollist;
4641 }
4642
4643 sub vm_stop_cleanup {
4644     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4645
4646     eval {
4647
4648         if (!$keepActive) {
4649             my $vollist = get_vm_volumes($conf);
4650             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4651         }
4652
4653         foreach my $ext (qw(mon qmp pid vnc qga)) {
4654             unlink "/var/run/qemu-server/${vmid}.$ext";
4655         }
4656
4657         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4658     };
4659     warn $@ if $@; # avoid errors - just warn
4660 }
4661
4662 # Note: use $nockeck to skip tests if VM configuration file exists.
4663 # We need that when migration VMs to other nodes (files already moved)
4664 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4665 sub vm_stop {
4666     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4667
4668     $force = 1 if !defined($force) && !$shutdown;
4669
4670     if ($migratedfrom){
4671         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4672         kill 15, $pid if $pid;
4673         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4674         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4675         return;
4676     }
4677
4678     PVE::QemuConfig->lock_config($vmid, sub {
4679
4680         my $pid = check_running($vmid, $nocheck);
4681         return if !$pid;
4682
4683         my $conf;
4684         if (!$nocheck) {
4685             $conf = PVE::QemuConfig->load_config($vmid);
4686             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4687             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4688                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4689                 $timeout = $opts->{down} if $opts->{down};
4690             }
4691         }
4692
4693         $timeout = 60 if !defined($timeout);
4694
4695         eval {
4696             if ($shutdown) {
4697                 if (defined($conf) && $conf->{agent}) {
4698                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4699                 } else {
4700                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4701                 }
4702             } else {
4703                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4704             }
4705         };
4706         my $err = $@;
4707
4708         if (!$err) {
4709             my $count = 0;
4710             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4711                 $count++;
4712                 sleep 1;
4713             }
4714
4715             if ($count >= $timeout) {
4716                 if ($force) {
4717                     warn "VM still running - terminating now with SIGTERM\n";
4718                     kill 15, $pid;
4719                 } else {
4720                     die "VM quit/powerdown failed - got timeout\n";
4721                 }
4722             } else {
4723                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4724                 return;
4725             }
4726         } else {
4727             if ($force) {
4728                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4729                 kill 15, $pid;
4730             } else {
4731                 die "VM quit/powerdown failed\n";
4732             }
4733         }
4734
4735         # wait again
4736         $timeout = 10;
4737
4738         my $count = 0;
4739         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4740             $count++;
4741             sleep 1;
4742         }
4743
4744         if ($count >= $timeout) {
4745             warn "VM still running - terminating now with SIGKILL\n";
4746             kill 9, $pid;
4747             sleep 1;
4748         }
4749
4750         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4751    });
4752 }
4753
4754 sub vm_suspend {
4755     my ($vmid, $skiplock) = @_;
4756
4757     PVE::QemuConfig->lock_config($vmid, sub {
4758
4759         my $conf = PVE::QemuConfig->load_config($vmid);
4760
4761         PVE::QemuConfig->check_lock($conf)
4762             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4763
4764         vm_mon_cmd($vmid, "stop");
4765     });
4766 }
4767
4768 sub vm_resume {
4769     my ($vmid, $skiplock, $nocheck) = @_;
4770
4771     PVE::QemuConfig->lock_config($vmid, sub {
4772
4773         if (!$nocheck) {
4774
4775             my $conf = PVE::QemuConfig->load_config($vmid);
4776
4777             PVE::QemuConfig->check_lock($conf)
4778                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4779
4780             vm_mon_cmd($vmid, "cont");
4781
4782         } else {
4783             vm_mon_cmd_nocheck($vmid, "cont");
4784         }
4785     });
4786 }
4787
4788 sub vm_sendkey {
4789     my ($vmid, $skiplock, $key) = @_;
4790
4791     PVE::QemuConfig->lock_config($vmid, sub {
4792
4793         my $conf = PVE::QemuConfig->load_config($vmid);
4794
4795         # there is no qmp command, so we use the human monitor command
4796         vm_human_monitor_command($vmid, "sendkey $key");
4797     });
4798 }
4799
4800 sub vm_destroy {
4801     my ($storecfg, $vmid, $skiplock) = @_;
4802
4803     PVE::QemuConfig->lock_config($vmid, sub {
4804
4805         my $conf = PVE::QemuConfig->load_config($vmid);
4806
4807         if (!check_running($vmid)) {
4808             destroy_vm($storecfg, $vmid, undef, $skiplock);
4809         } else {
4810             die "VM $vmid is running - destroy failed\n";
4811         }
4812     });
4813 }
4814
4815 # pci helpers
4816
4817 sub file_write {
4818     my ($filename, $buf) = @_;
4819
4820     my $fh = IO::File->new($filename, "w");
4821     return undef if !$fh;
4822
4823     my $res = print $fh $buf;
4824
4825     $fh->close();
4826
4827     return $res;
4828 }
4829
4830 sub pci_device_info {
4831     my ($name) = @_;
4832
4833     my $res;
4834
4835     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4836     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4837
4838     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4839     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4840
4841     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4842     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4843
4844     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4845     return undef if !defined($product) || $product !~ s/^0x//;
4846
4847     $res = {
4848         name => $name,
4849         vendor => $vendor,
4850         product => $product,
4851         domain => $domain,
4852         bus => $bus,
4853         slot => $slot,
4854         func => $func,
4855         irq => $irq,
4856         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4857     };
4858
4859     return $res;
4860 }
4861
4862 sub pci_dev_reset {
4863     my ($dev) = @_;
4864
4865     my $name = $dev->{name};
4866
4867     my $fn = "$pcisysfs/devices/$name/reset";
4868
4869     return file_write($fn, "1");
4870 }
4871
4872 sub pci_dev_bind_to_vfio {
4873     my ($dev) = @_;
4874
4875     my $name = $dev->{name};
4876
4877     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4878
4879     if (!-d $vfio_basedir) {
4880         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4881     }
4882     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4883
4884     my $testdir = "$vfio_basedir/$name";
4885     return 1 if -d $testdir;
4886
4887     my $data = "$dev->{vendor} $dev->{product}";
4888     return undef if !file_write("$vfio_basedir/new_id", $data);
4889
4890     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4891     if (!file_write($fn, $name)) {
4892         return undef if -f $fn;
4893     }
4894
4895     $fn = "$vfio_basedir/bind";
4896     if (! -d $testdir) {
4897         return undef if !file_write($fn, $name);
4898     }
4899
4900     return -d $testdir;
4901 }
4902
4903 sub pci_dev_group_bind_to_vfio {
4904     my ($pciid) = @_;
4905
4906     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4907
4908     if (!-d $vfio_basedir) {
4909         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4910     }
4911     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4912
4913     # get IOMMU group devices
4914     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4915       my @devs = grep /^0000:/, readdir($D);
4916     closedir($D);
4917
4918     foreach my $pciid (@devs) {
4919         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4920
4921         # pci bridges, switches or root ports are not supported
4922         # they have a pci_bus subdirectory so skip them
4923         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4924
4925         my $info = pci_device_info($1);
4926         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4927     }
4928
4929     return 1;
4930 }
4931
4932 sub print_pci_addr {
4933     my ($id, $bridges) = @_;
4934
4935     my $res = '';
4936     my $devices = {
4937         piix3 => { bus => 0, addr => 1 },
4938         #addr2 : first videocard
4939         balloon0 => { bus => 0, addr => 3 },
4940         watchdog => { bus => 0, addr => 4 },
4941         scsihw0 => { bus => 0, addr => 5 },
4942         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4943         scsihw1 => { bus => 0, addr => 6 },
4944         ahci0 => { bus => 0, addr => 7 },
4945         qga0 => { bus => 0, addr => 8 },
4946         spice => { bus => 0, addr => 9 },
4947         virtio0 => { bus => 0, addr => 10 },
4948         virtio1 => { bus => 0, addr => 11 },
4949         virtio2 => { bus => 0, addr => 12 },
4950         virtio3 => { bus => 0, addr => 13 },
4951         virtio4 => { bus => 0, addr => 14 },
4952         virtio5 => { bus => 0, addr => 15 },
4953         hostpci0 => { bus => 0, addr => 16 },
4954         hostpci1 => { bus => 0, addr => 17 },
4955         net0 => { bus => 0, addr => 18 },
4956         net1 => { bus => 0, addr => 19 },
4957         net2 => { bus => 0, addr => 20 },
4958         net3 => { bus => 0, addr => 21 },
4959         net4 => { bus => 0, addr => 22 },
4960         net5 => { bus => 0, addr => 23 },
4961         vga1 => { bus => 0, addr => 24 },
4962         vga2 => { bus => 0, addr => 25 },
4963         vga3 => { bus => 0, addr => 26 },
4964         hostpci2 => { bus => 0, addr => 27 },
4965         hostpci3 => { bus => 0, addr => 28 },
4966         #addr29 : usb-host (pve-usb.cfg)
4967         'pci.1' => { bus => 0, addr => 30 },
4968         'pci.2' => { bus => 0, addr => 31 },
4969         'net6' => { bus => 1, addr => 1 },
4970         'net7' => { bus => 1, addr => 2 },
4971         'net8' => { bus => 1, addr => 3 },
4972         'net9' => { bus => 1, addr => 4 },
4973         'net10' => { bus => 1, addr => 5 },
4974         'net11' => { bus => 1, addr => 6 },
4975         'net12' => { bus => 1, addr => 7 },
4976         'net13' => { bus => 1, addr => 8 },
4977         'net14' => { bus => 1, addr => 9 },
4978         'net15' => { bus => 1, addr => 10 },
4979         'net16' => { bus => 1, addr => 11 },
4980         'net17' => { bus => 1, addr => 12 },
4981         'net18' => { bus => 1, addr => 13 },
4982         'net19' => { bus => 1, addr => 14 },
4983         'net20' => { bus => 1, addr => 15 },
4984         'net21' => { bus => 1, addr => 16 },
4985         'net22' => { bus => 1, addr => 17 },
4986         'net23' => { bus => 1, addr => 18 },
4987         'net24' => { bus => 1, addr => 19 },
4988         'net25' => { bus => 1, addr => 20 },
4989         'net26' => { bus => 1, addr => 21 },
4990         'net27' => { bus => 1, addr => 22 },
4991         'net28' => { bus => 1, addr => 23 },
4992         'net29' => { bus => 1, addr => 24 },
4993         'net30' => { bus => 1, addr => 25 },
4994         'net31' => { bus => 1, addr => 26 },
4995         'xhci' => { bus => 1, addr => 27 },
4996         'virtio6' => { bus => 2, addr => 1 },
4997         'virtio7' => { bus => 2, addr => 2 },
4998         'virtio8' => { bus => 2, addr => 3 },
4999         'virtio9' => { bus => 2, addr => 4 },
5000         'virtio10' => { bus => 2, addr => 5 },
5001         'virtio11' => { bus => 2, addr => 6 },
5002         'virtio12' => { bus => 2, addr => 7 },
5003         'virtio13' => { bus => 2, addr => 8 },
5004         'virtio14' => { bus => 2, addr => 9 },
5005         'virtio15' => { bus => 2, addr => 10 },
5006         'virtioscsi0' => { bus => 3, addr => 1 },
5007         'virtioscsi1' => { bus => 3, addr => 2 },
5008         'virtioscsi2' => { bus => 3, addr => 3 },
5009         'virtioscsi3' => { bus => 3, addr => 4 },
5010         'virtioscsi4' => { bus => 3, addr => 5 },
5011         'virtioscsi5' => { bus => 3, addr => 6 },
5012         'virtioscsi6' => { bus => 3, addr => 7 },
5013         'virtioscsi7' => { bus => 3, addr => 8 },
5014         'virtioscsi8' => { bus => 3, addr => 9 },
5015         'virtioscsi9' => { bus => 3, addr => 10 },
5016         'virtioscsi10' => { bus => 3, addr => 11 },
5017         'virtioscsi11' => { bus => 3, addr => 12 },
5018         'virtioscsi12' => { bus => 3, addr => 13 },
5019         'virtioscsi13' => { bus => 3, addr => 14 },
5020         'virtioscsi14' => { bus => 3, addr => 15 },
5021         'virtioscsi15' => { bus => 3, addr => 16 },
5022         'virtioscsi16' => { bus => 3, addr => 17 },
5023         'virtioscsi17' => { bus => 3, addr => 18 },
5024         'virtioscsi18' => { bus => 3, addr => 19 },
5025         'virtioscsi19' => { bus => 3, addr => 20 },
5026         'virtioscsi20' => { bus => 3, addr => 21 },
5027         'virtioscsi21' => { bus => 3, addr => 22 },
5028         'virtioscsi22' => { bus => 3, addr => 23 },
5029         'virtioscsi23' => { bus => 3, addr => 24 },
5030         'virtioscsi24' => { bus => 3, addr => 25 },
5031         'virtioscsi25' => { bus => 3, addr => 26 },
5032         'virtioscsi26' => { bus => 3, addr => 27 },
5033         'virtioscsi27' => { bus => 3, addr => 28 },
5034         'virtioscsi28' => { bus => 3, addr => 29 },
5035         'virtioscsi29' => { bus => 3, addr => 30 },
5036         'virtioscsi30' => { bus => 3, addr => 31 },
5037
5038     };
5039
5040     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5041            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5042            my $bus = $devices->{$id}->{bus};
5043            $res = ",bus=pci.$bus,addr=$addr";
5044            $bridges->{$bus} = 1 if $bridges;
5045     }
5046     return $res;
5047
5048 }
5049
5050 sub print_pcie_addr {
5051     my ($id) = @_;
5052
5053     my $res = '';
5054     my $devices = {
5055         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5056         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5057         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5058         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5059     };
5060
5061     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5062            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5063            my $bus = $devices->{$id}->{bus};
5064            $res = ",bus=$bus,addr=$addr";
5065     }
5066     return $res;
5067
5068 }
5069
5070 # vzdump restore implementaion
5071
5072 sub tar_archive_read_firstfile {
5073     my $archive = shift;
5074
5075     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5076
5077     # try to detect archive type first
5078     my $pid = open (TMP, "tar tf '$archive'|") ||
5079         die "unable to open file '$archive'\n";
5080     my $firstfile = <TMP>;
5081     kill 15, $pid;
5082     close TMP;
5083
5084     die "ERROR: archive contaions no data\n" if !$firstfile;
5085     chomp $firstfile;
5086
5087     return $firstfile;
5088 }
5089
5090 sub tar_restore_cleanup {
5091     my ($storecfg, $statfile) = @_;
5092
5093     print STDERR "starting cleanup\n";
5094
5095     if (my $fd = IO::File->new($statfile, "r")) {
5096         while (defined(my $line = <$fd>)) {
5097             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5098                 my $volid = $2;
5099                 eval {
5100                     if ($volid =~ m|^/|) {
5101                         unlink $volid || die 'unlink failed\n';
5102                     } else {
5103                         PVE::Storage::vdisk_free($storecfg, $volid);
5104                     }
5105                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5106                 };
5107                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5108             } else {
5109                 print STDERR "unable to parse line in statfile - $line";
5110             }
5111         }
5112         $fd->close();
5113     }
5114 }
5115
5116 sub restore_archive {
5117     my ($archive, $vmid, $user, $opts) = @_;
5118
5119     my $format = $opts->{format};
5120     my $comp;
5121
5122     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5123         $format = 'tar' if !$format;
5124         $comp = 'gzip';
5125     } elsif ($archive =~ m/\.tar$/) {
5126         $format = 'tar' if !$format;
5127     } elsif ($archive =~ m/.tar.lzo$/) {
5128         $format = 'tar' if !$format;
5129         $comp = 'lzop';
5130     } elsif ($archive =~ m/\.vma$/) {
5131         $format = 'vma' if !$format;
5132     } elsif ($archive =~ m/\.vma\.gz$/) {
5133         $format = 'vma' if !$format;
5134         $comp = 'gzip';
5135     } elsif ($archive =~ m/\.vma\.lzo$/) {
5136         $format = 'vma' if !$format;
5137         $comp = 'lzop';
5138     } else {
5139         $format = 'vma' if !$format; # default
5140     }
5141
5142     # try to detect archive format
5143     if ($format eq 'tar') {
5144         return restore_tar_archive($archive, $vmid, $user, $opts);
5145     } else {
5146         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5147     }
5148 }
5149
5150 sub restore_update_config_line {
5151     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5152
5153     return if $line =~ m/^\#qmdump\#/;
5154     return if $line =~ m/^\#vzdump\#/;
5155     return if $line =~ m/^lock:/;
5156     return if $line =~ m/^unused\d+:/;
5157     return if $line =~ m/^parent:/;
5158     return if $line =~ m/^template:/; # restored VM is never a template
5159
5160     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5161         # try to convert old 1.X settings
5162         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5163         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5164             my ($model, $macaddr) = split(/\=/, $devconfig);
5165             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5166             my $net = {
5167                 model => $model,
5168                 bridge => "vmbr$ind",
5169                 macaddr => $macaddr,
5170             };
5171             my $netstr = print_net($net);
5172
5173             print $outfd "net$cookie->{netcount}: $netstr\n";
5174             $cookie->{netcount}++;
5175         }
5176     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5177         my ($id, $netstr) = ($1, $2);
5178         my $net = parse_net($netstr);
5179         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5180         $netstr = print_net($net);
5181         print $outfd "$id: $netstr\n";
5182     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5183         my $virtdev = $1;
5184         my $value = $3;
5185         my $di = parse_drive($virtdev, $value);
5186         if (defined($di->{backup}) && !$di->{backup}) {
5187             print $outfd "#$line";
5188         } elsif ($map->{$virtdev}) {
5189             delete $di->{format}; # format can change on restore
5190             $di->{file} = $map->{$virtdev};
5191             $value = print_drive($vmid, $di);
5192             print $outfd "$virtdev: $value\n";
5193         } else {
5194             print $outfd $line;
5195         }
5196     } else {
5197         print $outfd $line;
5198     }
5199 }
5200
5201 sub scan_volids {
5202     my ($cfg, $vmid) = @_;
5203
5204     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5205
5206     my $volid_hash = {};
5207     foreach my $storeid (keys %$info) {
5208         foreach my $item (@{$info->{$storeid}}) {
5209             next if !($item->{volid} && $item->{size});
5210             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5211             $volid_hash->{$item->{volid}} = $item;
5212         }
5213     }
5214
5215     return $volid_hash;
5216 }
5217
5218 sub is_volume_in_use {
5219     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5220
5221     my $path = PVE::Storage::path($storecfg, $volid);
5222
5223     my $scan_config = sub {
5224         my ($cref, $snapname) = @_;
5225
5226         foreach my $key (keys %$cref) {
5227             my $value = $cref->{$key};
5228             if (is_valid_drivename($key)) {
5229                 next if $skip_drive && $key eq $skip_drive;
5230                 my $drive = parse_drive($key, $value);
5231                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5232                 return 1 if $volid eq $drive->{file};
5233                 if ($drive->{file} =~ m!^/!) {
5234                     return 1 if $drive->{file} eq $path;
5235                 } else {
5236                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5237                     next if !$storeid;
5238                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5239                     next if !$scfg;
5240                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5241                 }
5242             }
5243         }
5244
5245         return 0;
5246     };
5247
5248     return 1 if &$scan_config($conf);
5249
5250     undef $skip_drive;
5251
5252     foreach my $snapname (keys %{$conf->{snapshots}}) {
5253         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5254     }
5255
5256     return 0;
5257 }
5258
5259 sub update_disksize {
5260     my ($vmid, $conf, $volid_hash) = @_;
5261
5262     my $changes;
5263
5264     my $used = {};
5265
5266     # Note: it is allowed to define multiple storages with same path (alias), so
5267     # we need to check both 'volid' and real 'path' (two different volid can point
5268     # to the same path).
5269
5270     my $usedpath = {};
5271
5272     # update size info
5273     foreach my $opt (keys %$conf) {
5274         if (is_valid_drivename($opt)) {
5275             my $drive = parse_drive($opt, $conf->{$opt});
5276             my $volid = $drive->{file};
5277             next if !$volid;
5278
5279             $used->{$volid} = 1;
5280             if ($volid_hash->{$volid} &&
5281                 (my $path = $volid_hash->{$volid}->{path})) {
5282                 $usedpath->{$path} = 1;
5283             }
5284
5285             next if drive_is_cdrom($drive);
5286             next if !$volid_hash->{$volid};
5287
5288             $drive->{size} = $volid_hash->{$volid}->{size};
5289             my $new = print_drive($vmid, $drive);
5290             if ($new ne $conf->{$opt}) {
5291                 $changes = 1;
5292                 $conf->{$opt} = $new;
5293             }
5294         }
5295     }
5296
5297     # remove 'unusedX' entry if volume is used
5298     foreach my $opt (keys %$conf) {
5299         next if $opt !~ m/^unused\d+$/;
5300         my $volid = $conf->{$opt};
5301         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5302         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5303             $changes = 1;
5304             delete $conf->{$opt};
5305         }
5306     }
5307
5308     foreach my $volid (sort keys %$volid_hash) {
5309         next if $volid =~ m/vm-$vmid-state-/;
5310         next if $used->{$volid};
5311         my $path = $volid_hash->{$volid}->{path};
5312         next if !$path; # just to be sure
5313         next if $usedpath->{$path};
5314         $changes = 1;
5315         PVE::QemuConfig->add_unused_volume($conf, $volid);
5316         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5317     }
5318
5319     return $changes;
5320 }
5321
5322 sub rescan {
5323     my ($vmid, $nolock) = @_;
5324
5325     my $cfg = PVE::Storage::config();
5326
5327     my $volid_hash = scan_volids($cfg, $vmid);
5328
5329     my $updatefn =  sub {
5330         my ($vmid) = @_;
5331
5332         my $conf = PVE::QemuConfig->load_config($vmid);
5333
5334         PVE::QemuConfig->check_lock($conf);
5335
5336         my $vm_volids = {};
5337         foreach my $volid (keys %$volid_hash) {
5338             my $info = $volid_hash->{$volid};
5339             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5340         }
5341
5342         my $changes = update_disksize($vmid, $conf, $vm_volids);
5343
5344         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5345     };
5346
5347     if (defined($vmid)) {
5348         if ($nolock) {
5349             &$updatefn($vmid);
5350         } else {
5351             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5352         }
5353     } else {
5354         my $vmlist = config_list();
5355         foreach my $vmid (keys %$vmlist) {
5356             if ($nolock) {
5357                 &$updatefn($vmid);
5358             } else {
5359                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5360             }
5361         }
5362     }
5363 }
5364
5365 sub restore_vma_archive {
5366     my ($archive, $vmid, $user, $opts, $comp) = @_;
5367
5368     my $input = $archive eq '-' ? "<&STDIN" : undef;
5369     my $readfrom = $archive;
5370
5371     my $uncomp = '';
5372     if ($comp) {
5373         $readfrom = '-';
5374         my $qarchive = PVE::Tools::shellquote($archive);
5375         if ($comp eq 'gzip') {
5376             $uncomp = "zcat $qarchive|";
5377         } elsif ($comp eq 'lzop') {
5378             $uncomp = "lzop -d -c $qarchive|";
5379         } else {
5380             die "unknown compression method '$comp'\n";
5381         }
5382
5383     }
5384
5385     my $tmpdir = "/var/tmp/vzdumptmp$$";
5386     rmtree $tmpdir;
5387
5388     # disable interrupts (always do cleanups)
5389     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5390         warn "got interrupt - ignored\n";
5391     };
5392
5393     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5394     POSIX::mkfifo($mapfifo, 0600);
5395     my $fifofh;
5396
5397     my $openfifo = sub {
5398         open($fifofh, '>', $mapfifo) || die $!;
5399     };
5400
5401     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5402
5403     my $oldtimeout;
5404     my $timeout = 5;
5405
5406     my $devinfo = {};
5407
5408     my $rpcenv = PVE::RPCEnvironment::get();
5409
5410     my $conffile = PVE::QemuConfig->config_file($vmid);
5411     my $tmpfn = "$conffile.$$.tmp";
5412
5413     # Note: $oldconf is undef if VM does not exists
5414     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5415     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5416
5417     my $print_devmap = sub {
5418         my $virtdev_hash = {};
5419
5420         my $cfgfn = "$tmpdir/qemu-server.conf";
5421
5422         # we can read the config - that is already extracted
5423         my $fh = IO::File->new($cfgfn, "r") ||
5424             "unable to read qemu-server.conf - $!\n";
5425
5426         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5427         if (-f $fwcfgfn) {
5428             my $pve_firewall_dir = '/etc/pve/firewall';
5429             mkdir $pve_firewall_dir; # make sure the dir exists
5430             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5431         }
5432
5433         while (defined(my $line = <$fh>)) {
5434             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5435                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5436                 die "archive does not contain data for drive '$virtdev'\n"
5437                     if !$devinfo->{$devname};
5438                 if (defined($opts->{storage})) {
5439                     $storeid = $opts->{storage} || 'local';
5440                 } elsif (!$storeid) {
5441                     $storeid = 'local';
5442                 }
5443                 $format = 'raw' if !$format;
5444                 $devinfo->{$devname}->{devname} = $devname;
5445                 $devinfo->{$devname}->{virtdev} = $virtdev;
5446                 $devinfo->{$devname}->{format} = $format;
5447                 $devinfo->{$devname}->{storeid} = $storeid;
5448
5449                 # check permission on storage
5450                 my $pool = $opts->{pool}; # todo: do we need that?
5451                 if ($user ne 'root@pam') {
5452                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5453                 }
5454
5455                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5456             }
5457         }
5458
5459         foreach my $devname (keys %$devinfo) {
5460             die "found no device mapping information for device '$devname'\n"
5461                 if !$devinfo->{$devname}->{virtdev};
5462         }
5463
5464         my $cfg = PVE::Storage::config();
5465
5466         # create empty/temp config
5467         if ($oldconf) {
5468             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5469             foreach_drive($oldconf, sub {
5470                 my ($ds, $drive) = @_;
5471
5472                 return if drive_is_cdrom($drive);
5473
5474                 my $volid = $drive->{file};
5475
5476                 return if !$volid || $volid =~ m|^/|;
5477
5478                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5479                 return if !$path || !$owner || ($owner != $vmid);
5480
5481                 # Note: only delete disk we want to restore
5482                 # other volumes will become unused
5483                 if ($virtdev_hash->{$ds}) {
5484                     PVE::Storage::vdisk_free($cfg, $volid);
5485                 }
5486             });
5487
5488             # delete vmstate files
5489             # since after the restore we have no snapshots anymore
5490             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5491                 my $snap = $oldconf->{snapshots}->{$snapname};
5492                 if ($snap->{vmstate}) {
5493                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5494                     if (my $err = $@) {
5495                         warn $err;
5496                     }
5497                 }
5498             }
5499         }
5500
5501         my $map = {};
5502         foreach my $virtdev (sort keys %$virtdev_hash) {
5503             my $d = $virtdev_hash->{$virtdev};
5504             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5505             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5506
5507             # test if requested format is supported
5508             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5509             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5510             $d->{format} = $defFormat if !$supported;
5511
5512             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5513                                                   $d->{format}, undef, $alloc_size);
5514             print STDERR "new volume ID is '$volid'\n";
5515             $d->{volid} = $volid;
5516             my $path = PVE::Storage::path($cfg, $volid);
5517
5518             PVE::Storage::activate_volumes($cfg,[$volid]);
5519
5520             my $write_zeros = 1;
5521             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5522                 $write_zeros = 0;
5523             }
5524
5525             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5526
5527             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5528             $map->{$virtdev} = $volid;
5529         }
5530
5531         $fh->seek(0, 0) || die "seek failed - $!\n";
5532
5533         my $outfd = new IO::File ($tmpfn, "w") ||
5534             die "unable to write config for VM $vmid\n";
5535
5536         my $cookie = { netcount => 0 };
5537         while (defined(my $line = <$fh>)) {
5538             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5539         }
5540
5541         $fh->close();
5542         $outfd->close();
5543     };
5544
5545     eval {
5546         # enable interrupts
5547         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5548             die "interrupted by signal\n";
5549         };
5550         local $SIG{ALRM} = sub { die "got timeout\n"; };
5551
5552         $oldtimeout = alarm($timeout);
5553
5554         my $parser = sub {
5555             my $line = shift;
5556
5557             print "$line\n";
5558
5559             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5560                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5561                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5562             } elsif ($line =~ m/^CTIME: /) {
5563                 # we correctly received the vma config, so we can disable
5564                 # the timeout now for disk allocation (set to 10 minutes, so
5565                 # that we always timeout if something goes wrong)
5566                 alarm(600);
5567                 &$print_devmap();
5568                 print $fifofh "done\n";
5569                 my $tmp = $oldtimeout || 0;
5570                 $oldtimeout = undef;
5571                 alarm($tmp);
5572                 close($fifofh);
5573             }
5574         };
5575
5576         print "restore vma archive: $cmd\n";
5577         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5578     };
5579     my $err = $@;
5580
5581     alarm($oldtimeout) if $oldtimeout;
5582
5583     my $vollist = [];
5584     foreach my $devname (keys %$devinfo) {
5585         my $volid = $devinfo->{$devname}->{volid};
5586         push @$vollist, $volid if $volid;
5587     }
5588
5589     my $cfg = PVE::Storage::config();
5590     PVE::Storage::deactivate_volumes($cfg, $vollist);
5591
5592     unlink $mapfifo;
5593
5594     if ($err) {
5595         rmtree $tmpdir;
5596         unlink $tmpfn;
5597
5598         foreach my $devname (keys %$devinfo) {
5599             my $volid = $devinfo->{$devname}->{volid};
5600             next if !$volid;
5601             eval {
5602                 if ($volid =~ m|^/|) {
5603                     unlink $volid || die 'unlink failed\n';
5604                 } else {
5605                     PVE::Storage::vdisk_free($cfg, $volid);
5606                 }
5607                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5608             };
5609             print STDERR "unable to cleanup '$volid' - $@" if $@;
5610         }
5611         die $err;
5612     }
5613
5614     rmtree $tmpdir;
5615
5616     rename($tmpfn, $conffile) ||
5617         die "unable to commit configuration file '$conffile'\n";
5618
5619     PVE::Cluster::cfs_update(); # make sure we read new file
5620
5621     eval { rescan($vmid, 1); };
5622     warn $@ if $@;
5623 }
5624
5625 sub restore_tar_archive {
5626     my ($archive, $vmid, $user, $opts) = @_;
5627
5628     if ($archive ne '-') {
5629         my $firstfile = tar_archive_read_firstfile($archive);
5630         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5631             if $firstfile ne 'qemu-server.conf';
5632     }
5633
5634     my $storecfg = PVE::Storage::config();
5635
5636     # destroy existing data - keep empty config
5637     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5638     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5639
5640     my $tocmd = "/usr/lib/qemu-server/qmextract";
5641
5642     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5643     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5644     $tocmd .= ' --prealloc' if $opts->{prealloc};
5645     $tocmd .= ' --info' if $opts->{info};
5646
5647     # tar option "xf" does not autodetect compression when read from STDIN,
5648     # so we pipe to zcat
5649     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5650         PVE::Tools::shellquote("--to-command=$tocmd");
5651
5652     my $tmpdir = "/var/tmp/vzdumptmp$$";
5653     mkpath $tmpdir;
5654
5655     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5656     local $ENV{VZDUMP_VMID} = $vmid;
5657     local $ENV{VZDUMP_USER} = $user;
5658
5659     my $conffile = PVE::QemuConfig->config_file($vmid);
5660     my $tmpfn = "$conffile.$$.tmp";
5661
5662     # disable interrupts (always do cleanups)
5663     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5664         print STDERR "got interrupt - ignored\n";
5665     };
5666
5667     eval {
5668         # enable interrupts
5669         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5670             die "interrupted by signal\n";
5671         };
5672
5673         if ($archive eq '-') {
5674             print "extracting archive from STDIN\n";
5675             run_command($cmd, input => "<&STDIN");
5676         } else {
5677             print "extracting archive '$archive'\n";
5678             run_command($cmd);
5679         }
5680
5681         return if $opts->{info};
5682
5683         # read new mapping
5684         my $map = {};
5685         my $statfile = "$tmpdir/qmrestore.stat";
5686         if (my $fd = IO::File->new($statfile, "r")) {
5687             while (defined (my $line = <$fd>)) {
5688                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5689                     $map->{$1} = $2 if $1;
5690                 } else {
5691                     print STDERR "unable to parse line in statfile - $line\n";
5692                 }
5693             }
5694             $fd->close();
5695         }
5696
5697         my $confsrc = "$tmpdir/qemu-server.conf";
5698
5699         my $srcfd = new IO::File($confsrc, "r") ||
5700             die "unable to open file '$confsrc'\n";
5701
5702         my $outfd = new IO::File ($tmpfn, "w") ||
5703             die "unable to write config for VM $vmid\n";
5704
5705         my $cookie = { netcount => 0 };
5706         while (defined (my $line = <$srcfd>)) {
5707             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5708         }
5709
5710         $srcfd->close();
5711         $outfd->close();
5712     };
5713     my $err = $@;
5714
5715     if ($err) {
5716
5717         unlink $tmpfn;
5718
5719         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5720
5721         die $err;
5722     }
5723
5724     rmtree $tmpdir;
5725
5726     rename $tmpfn, $conffile ||
5727         die "unable to commit configuration file '$conffile'\n";
5728
5729     PVE::Cluster::cfs_update(); # make sure we read new file
5730
5731     eval { rescan($vmid, 1); };
5732     warn $@ if $@;
5733 };
5734
5735 sub foreach_writable_storage {
5736     my ($conf, $func) = @_;
5737
5738     my $sidhash = {};
5739
5740     foreach my $ds (keys %$conf) {
5741         next if !is_valid_drivename($ds);
5742
5743         my $drive = parse_drive($ds, $conf->{$ds});
5744         next if !$drive;
5745         next if drive_is_cdrom($drive);
5746
5747         my $volid = $drive->{file};
5748
5749         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5750         $sidhash->{$sid} = $sid if $sid;
5751     }
5752
5753     foreach my $sid (sort keys %$sidhash) {
5754         &$func($sid);
5755     }
5756 }
5757
5758 sub do_snapshots_with_qemu {
5759     my ($storecfg, $volid) = @_;
5760
5761     my $storage_name = PVE::Storage::parse_volume_id($volid);
5762
5763     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5764         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5765         return 1;
5766     }
5767
5768     if ($volid =~ m/\.(qcow2|qed)$/){
5769         return 1;
5770     }
5771
5772     return undef;
5773 }
5774
5775 sub qga_check_running {
5776     my ($vmid) = @_;
5777
5778     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5779     if ($@) {
5780         warn "Qemu Guest Agent are not running - $@";
5781         return 0;
5782     }
5783     return 1;
5784 }
5785
5786 sub template_create {
5787     my ($vmid, $conf, $disk) = @_;
5788
5789     my $storecfg = PVE::Storage::config();
5790
5791     foreach_drive($conf, sub {
5792         my ($ds, $drive) = @_;
5793
5794         return if drive_is_cdrom($drive);
5795         return if $disk && $ds ne $disk;
5796
5797         my $volid = $drive->{file};
5798         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5799
5800         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5801         $drive->{file} = $voliddst;
5802         $conf->{$ds} = print_drive($vmid, $drive);
5803         PVE::QemuConfig->write_config($vmid, $conf);
5804     });
5805 }
5806
5807 sub qemu_img_convert {
5808     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5809
5810     my $storecfg = PVE::Storage::config();
5811     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5812     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5813
5814     if ($src_storeid && $dst_storeid) {
5815
5816         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5817
5818         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5819         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5820
5821         my $src_format = qemu_img_format($src_scfg, $src_volname);
5822         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5823
5824         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5825         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5826
5827         my $cmd = [];
5828         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5829         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5830         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5831         if ($is_zero_initialized) {
5832             push @$cmd, "zeroinit:$dst_path";
5833         } else {
5834             push @$cmd, $dst_path;
5835         }
5836
5837         my $parser = sub {
5838             my $line = shift;
5839             if($line =~ m/\((\S+)\/100\%\)/){
5840                 my $percent = $1;
5841                 my $transferred = int($size * $percent / 100);
5842                 my $remaining = $size - $transferred;
5843
5844                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5845             }
5846
5847         };
5848
5849         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5850         my $err = $@;
5851         die "copy failed: $err" if $err;
5852     }
5853 }
5854
5855 sub qemu_img_format {
5856     my ($scfg, $volname) = @_;
5857
5858     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5859         return $1;
5860     } else {
5861         return "raw";
5862     }
5863 }
5864
5865 sub qemu_drive_mirror {
5866     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5867
5868     my $storecfg = PVE::Storage::config();
5869     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5870
5871     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5872
5873     my $format = qemu_img_format($dst_scfg, $dst_volname);
5874
5875     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5876
5877     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5878
5879     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5880     $opts->{format} = $format if $format;
5881
5882     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5883
5884     eval {
5885     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5886         while (1) {
5887             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5888             my $stat = @$stats[0];
5889             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5890             die "error job is not mirroring" if $stat->{type} ne "mirror";
5891
5892             my $busy = $stat->{busy};
5893             my $ready = $stat->{ready};
5894
5895             if (my $total = $stat->{len}) {
5896                 my $transferred = $stat->{offset} || 0;
5897                 my $remaining = $total - $transferred;
5898                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5899
5900                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5901             }
5902
5903
5904             if ($stat->{ready} eq 'true') {
5905
5906                 last if $vmiddst != $vmid;
5907
5908                 # try to switch the disk if source and destination are on the same guest
5909                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5910                 last if !$@;
5911                 die $@ if $@ !~ m/cannot be completed/;
5912             }
5913             sleep 1;
5914         }
5915
5916
5917     };
5918     my $err = $@;
5919
5920     my $cancel_job = sub {
5921         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5922         while (1) {
5923             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5924             my $stat = @$stats[0];
5925             last if !$stat;
5926             sleep 1;
5927         }
5928     };
5929
5930     if ($err) {
5931         eval { &$cancel_job(); };
5932         die "mirroring error: $err";
5933     }
5934
5935     if ($vmiddst != $vmid) {
5936         # if we clone a disk for a new target vm, we don't switch the disk
5937         &$cancel_job(); # so we call block-job-cancel
5938     }
5939 }
5940
5941 sub clone_disk {
5942     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5943         $newvmid, $storage, $format, $full, $newvollist) = @_;
5944
5945     my $newvolid;
5946
5947     if (!$full) {
5948         print "create linked clone of drive $drivename ($drive->{file})\n";
5949         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5950         push @$newvollist, $newvolid;
5951     } else {
5952         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5953         $storeid = $storage if $storage;
5954
5955         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5956         if (!$format) {
5957             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5958             $format = qemu_img_format($scfg, $volname);
5959         }
5960
5961         # test if requested format is supported - else use default
5962         my $supported = grep { $_ eq $format } @$validFormats;
5963         $format = $defFormat if !$supported;
5964
5965         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5966
5967         print "create full clone of drive $drivename ($drive->{file})\n";
5968         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5969         push @$newvollist, $newvolid;
5970
5971         PVE::Storage::activate_volumes($storecfg, $newvollist);
5972
5973         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5974         if (!$running || $snapname) {
5975             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5976         } else {
5977             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5978         }
5979     }
5980
5981     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5982
5983     my $disk = $drive;
5984     $disk->{format} = undef;
5985     $disk->{file} = $newvolid;
5986     $disk->{size} = $size;
5987
5988     return $disk;
5989 }
5990
5991 # this only works if VM is running
5992 sub get_current_qemu_machine {
5993     my ($vmid) = @_;
5994
5995     my $cmd = { execute => 'query-machines', arguments => {} };
5996     my $res = vm_qmp_command($vmid, $cmd);
5997
5998     my ($current, $default);
5999     foreach my $e (@$res) {
6000         $default = $e->{name} if $e->{'is-default'};
6001         $current = $e->{name} if $e->{'is-current'};
6002     }
6003
6004     # fallback to the default machine if current is not supported by qemu
6005     return $current || $default || 'pc';
6006 }
6007
6008 sub qemu_machine_feature_enabled {
6009     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6010
6011     my $current_major;
6012     my $current_minor;
6013
6014     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6015
6016         $current_major = $3;
6017         $current_minor = $4;
6018
6019     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6020
6021         $current_major = $1;
6022         $current_minor = $2;
6023     }
6024
6025     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6026
6027
6028 }
6029
6030 sub qemu_machine_pxe {
6031     my ($vmid, $conf, $machine) = @_;
6032
6033     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6034
6035     foreach my $opt (keys %$conf) {
6036         next if $opt !~ m/^net(\d+)$/;
6037         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6038         next if !$net;
6039         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6040         return $machine.".pxe" if $romfile =~ m/pxe/;
6041         last;
6042     }
6043
6044     return $machine;
6045 }
6046
6047 sub qemu_use_old_bios_files {
6048     my ($machine_type) = @_;
6049
6050     return if !$machine_type;
6051
6052     my $use_old_bios_files = undef;
6053
6054     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6055         $machine_type = $1;
6056         $use_old_bios_files = 1;
6057     } else {
6058         my $kvmver = kvm_user_version();
6059         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6060         # load new efi bios files on migration. So this hack is required to allow
6061         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6062         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6063         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6064     }
6065
6066     return ($use_old_bios_files, $machine_type);
6067 }
6068
6069 sub lspci {
6070
6071     my $devices = {};
6072
6073     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6074             my (undef, $id, $function) = @_;
6075             my $res = { id => $id, function => $function};
6076             push @{$devices->{$id}}, $res;
6077     });
6078
6079     return $devices;
6080 }
6081
6082 sub vm_iothreads_list {
6083     my ($vmid) = @_;
6084
6085     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6086
6087     my $iothreads = {};
6088     foreach my $iothread (@$res) {
6089         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6090     }
6091
6092     return $iothreads;
6093 }
6094
6095 sub scsihw_infos {
6096     my ($conf, $drive) = @_;
6097
6098     my $maxdev = 0;
6099
6100     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6101         $maxdev = 7;
6102     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6103         $maxdev = 1;
6104     } else {
6105         $maxdev = 256;
6106     }
6107
6108     my $controller = int($drive->{index} / $maxdev);
6109     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6110
6111     return ($maxdev, $controller, $controller_prefix);
6112 }
6113
6114 # bash completion helper
6115
6116 sub complete_backup_archives {
6117     my ($cmdname, $pname, $cvalue) = @_;
6118
6119     my $cfg = PVE::Storage::config();
6120
6121     my $storeid;
6122
6123     if ($cvalue =~ m/^([^:]+):/) {
6124         $storeid = $1;
6125     }
6126
6127     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6128
6129     my $res = [];
6130     foreach my $id (keys %$data) {
6131         foreach my $item (@{$data->{$id}}) {
6132             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6133             push @$res, $item->{volid} if defined($item->{volid});
6134         }
6135     }
6136
6137     return $res;
6138 }
6139
6140 my $complete_vmid_full = sub {
6141     my ($running) = @_;
6142
6143     my $idlist = vmstatus();
6144
6145     my $res = [];
6146
6147     foreach my $id (keys %$idlist) {
6148         my $d = $idlist->{$id};
6149         if (defined($running)) {
6150             next if $d->{template};
6151             next if $running && $d->{status} ne 'running';
6152             next if !$running && $d->{status} eq 'running';
6153         }
6154         push @$res, $id;
6155
6156     }
6157     return $res;
6158 };
6159
6160 sub complete_vmid {
6161     return &$complete_vmid_full();
6162 }
6163
6164 sub complete_vmid_stopped {
6165     return &$complete_vmid_full(0);
6166 }
6167
6168 sub complete_vmid_running {
6169     return &$complete_vmid_full(1);
6170 }
6171
6172 sub complete_storage {
6173
6174     my $cfg = PVE::Storage::config();
6175     my $ids = $cfg->{ids};
6176
6177     my $res = [];
6178     foreach my $sid (keys %$ids) {
6179         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6180         next if !$ids->{$sid}->{content}->{images};
6181         push @$res, $sid;
6182     }
6183
6184     return $res;
6185 }
6186
6187 1;