PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35 use URI::Escape;
  36
  37 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  38
  39 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  40
  41 # Note about locking: we use flock on the config file protect
  42 # against concurent actions.
  43 # Aditionaly, we have a 'lock' setting in the config file. This
  44 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  45 # allowed when such lock is set. But you can ignore this kind of
  46 # lock with the --skiplock flag.
  47
  48 cfs_register_file('/qemu-server/',
  49                   \&parse_vm_config,
  50                   \&write_vm_config);
  51
  52 PVE::JSONSchema::register_standard_option('skiplock', {
  53     description => "Ignore locks - only root is allowed to use this option.",
  54     type => 'boolean',
  55     optional => 1,
  56 });
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 #no warnings 'redefine';
  72
  73 sub cgroups_write {
  74    my ($controller, $vmid, $option, $value) = @_;
  75
  76    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  77    PVE::ProcFSTools::write_proc_entry($path, $value);
  78
  79 }
  80
  81 my $nodename = PVE::INotify::nodename();
  82
  83 mkdir "/etc/pve/nodes/$nodename";
  84 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  85 mkdir $confdir;
  86
  87 my $var_run_tmpdir = "/var/run/qemu-server";
  88 mkdir $var_run_tmpdir;
  89
  90 my $lock_dir = "/var/lock/qemu-server";
  91 mkdir $lock_dir;
  92
  93 my $pcisysfs = "/sys/bus/pci";
  94
  95 my $cpu_fmt = {
  96     cputype => {
  97         description => "Emulated CPU type.",
  98         type => 'string',
  99         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 100         format_description => 'cputype',
 101         default => 'kvm64',
 102         default_key => 1,
 103     },
 104     hidden => {
 105         description => "Do not identify as a KVM virtual machine.",
 106         type => 'boolean',
 107         optional => 1,
 108         default => 0
 109     },
 110 };
 111
 112 my $confdesc = {
 113     onboot => {
 114         optional => 1,
 115         type => 'boolean',
 116         description => "Specifies whether a VM will be started during system bootup.",
 117         default => 0,
 118     },
 119     autostart => {
 120         optional => 1,
 121         type => 'boolean',
 122         description => "Automatic restart after crash (currently ignored).",
 123         default => 0,
 124     },
 125     hotplug => {
 126         optional => 1,
 127         type => 'string', format => 'pve-hotplug-features',
 128         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 129         default => 'network,disk,usb',
 130     },
 131     reboot => {
 132         optional => 1,
 133         type => 'boolean',
 134         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 135         default => 1,
 136     },
 137     lock => {
 138         optional => 1,
 139         type => 'string',
 140         description => "Lock/unlock the VM.",
 141         enum => [qw(migrate backup snapshot rollback)],
 142     },
 143     cpulimit => {
 144         optional => 1,
 145         type => 'number',
 146         description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 147         minimum => 0,
 148         maximum => 128,
 149         default => 0,
 150     },
 151     cpuunits => {
 152         optional => 1,
 153         type => 'integer',
 154         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 155         minimum => 0,
 156         maximum => 500000,
 157         default => 1000,
 158     },
 159     memory => {
 160         optional => 1,
 161         type => 'integer',
 162         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 163         minimum => 16,
 164         default => 512,
 165     },
 166     balloon => {
 167         optional => 1,
 168         type => 'integer',
 169         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 170         minimum => 0,
 171     },
 172     shares => {
 173         optional => 1,
 174         type => 'integer',
 175         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 176         minimum => 0,
 177         maximum => 50000,
 178         default => 1000,
 179     },
 180     keyboard => {
 181         optional => 1,
 182         type => 'string',
 183         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 184         enum => PVE::Tools::kvmkeymaplist(),
 185         default => 'en-us',
 186     },
 187     name => {
 188         optional => 1,
 189         type => 'string', format => 'dns-name',
 190         description => "Set a name for the VM. Only used on the configuration web interface.",
 191     },
 192     scsihw => {
 193         optional => 1,
 194         type => 'string',
 195         description => "scsi controller model",
 196         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 197         default => 'lsi',
 198     },
 199     description => {
 200         optional => 1,
 201         type => 'string',
 202         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 203     },
 204     ostype => {
 205         optional => 1,
 206         type => 'string',
 207         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 208         description => <<EODESC,
 209 Used to enable special optimization/features for specific
 210 operating systems:
 211
 212 other  => unspecified OS
 213 wxp    => Microsoft Windows XP
 214 w2k    => Microsoft Windows 2000
 215 w2k3   => Microsoft Windows 2003
 216 w2k8   => Microsoft Windows 2008
 217 wvista => Microsoft Windows Vista
 218 win7   => Microsoft Windows 7
 219 win8   => Microsoft Windows 8/2012
 220 l24    => Linux 2.4 Kernel
 221 l26    => Linux 2.6/3.X Kernel
 222 solaris => solaris/opensolaris/openindiania kernel
 223
 224 other|l24|l26|solaris                       ... no special behaviour
 225 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 226 EODESC
 227     },
 228     boot => {
 229         optional => 1,
 230         type => 'string',
 231         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 232         pattern => '[acdn]{1,4}',
 233         default => 'cdn',
 234     },
 235     bootdisk => {
 236         optional => 1,
 237         type => 'string', format => 'pve-qm-bootdisk',
 238         description => "Enable booting from specified disk.",
 239         pattern => '(ide|sata|scsi|virtio)\d+',
 240     },
 241     smp => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "The number of CPUs. Please use option -sockets instead.",
 245         minimum => 1,
 246         default => 1,
 247     },
 248     sockets => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "The number of CPU sockets.",
 252         minimum => 1,
 253         default => 1,
 254     },
 255     cores => {
 256         optional => 1,
 257         type => 'integer',
 258         description => "The number of cores per socket.",
 259         minimum => 1,
 260         default => 1,
 261     },
 262     numa => {
 263         optional => 1,
 264         type => 'boolean',
 265         description => "Enable/disable NUMA.",
 266         default => 0,
 267     },
 268     vcpus => {
 269         optional => 1,
 270         type => 'integer',
 271         description => "Number of hotplugged vcpus.",
 272         minimum => 1,
 273         default => 0,
 274     },
 275     acpi => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Enable/disable ACPI.",
 279         default => 1,
 280     },
 281     agent => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Enable/disable Qemu GuestAgent.",
 285         default => 0,
 286     },
 287     kvm => {
 288         optional => 1,
 289         type => 'boolean',
 290         description => "Enable/disable KVM hardware virtualization.",
 291         default => 1,
 292     },
 293     tdf => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Enable/disable time drift fix.",
 297         default => 0,
 298     },
 299     localtime => {
 300         optional => 1,
 301         type => 'boolean',
 302         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 303     },
 304     freeze => {
 305         optional => 1,
 306         type => 'boolean',
 307         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 308     },
 309     vga => {
 310         optional => 1,
 311         type => 'string',
 312         description => "Select the VGA type. If you want to use high resolution" .
 313             " modes (>= 1280x1024x16) then you should use the options " .
 314             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 315             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 316             "display sever. For win* OS you can select how many independent " .
 317             "displays you want, Linux guests can add displays them self. " .
 318             "You can also run without any graphic card, using a serial device" .
 319             " as terminal.",
 320         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 321     },
 322     watchdog => {
 323         optional => 1,
 324         type => 'string', format => 'pve-qm-watchdog',
 325         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 326         description => "Create a virtual hardware watchdog device. Once enabled" .
 327             " (by a guest action), the watchdog must be periodically polled " .
 328             "by an agent inside the guest or else the watchdog will reset " .
 329             "the guest (or execute the respective action specified)",
 330     },
 331     startdate => {
 332         optional => 1,
 333         type => 'string',
 334         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 335         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 336         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 337         default => 'now',
 338     },
 339     startup =>  get_standard_option('pve-startup-order'),
 340     template => {
 341         optional => 1,
 342         type => 'boolean',
 343         description => "Enable/disable Template.",
 344         default => 0,
 345     },
 346     args => {
 347         optional => 1,
 348         type => 'string',
 349         description => <<EODESCR,
 350 NOTE: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 351
 352 args: -no-reboot -no-hpet
 353 EODESCR
 354     },
 355     tablet => {
 356         optional => 1,
 357         type => 'boolean',
 358         default => 1,
 359         description => "Enable/disable the USB tablet device. This device is " .
 360             "usually needed to allow absolute mouse positioning with VNC. " .
 361             "Else the mouse runs out of sync with normal VNC clients. " .
 362             "If you're running lots of console-only guests on one host, " .
 363             "you may consider disabling this to save some context switches. " .
 364             "This is turned off by default if you use spice (-vga=qxl).",
 365     },
 366     migrate_speed => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 370         minimum => 0,
 371         default => 0,
 372     },
 373     migrate_downtime => {
 374         optional => 1,
 375         type => 'number',
 376         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 377         minimum => 0,
 378         default => 0.1,
 379     },
 380     cdrom => {
 381         optional => 1,
 382         type => 'string', format => 'pve-qm-drive',
 383         typetext => 'volume',
 384         description => "This is an alias for option -ide2",
 385     },
 386     cpu => {
 387         optional => 1,
 388         description => "Emulated CPU type.",
 389         type => 'string',
 390         format => $cpu_fmt,
 391     },
 392     parent => get_standard_option('pve-snapshot-name', {
 393         optional => 1,
 394         description => "Parent snapshot name. This is used internally, and should not be modified.",
 395     }),
 396     snaptime => {
 397         optional => 1,
 398         description => "Timestamp for snapshots.",
 399         type => 'integer',
 400         minimum => 0,
 401     },
 402     vmstate => {
 403         optional => 1,
 404         type => 'string', format => 'pve-volume-id',
 405         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 406     },
 407     machine => {
 408         description => "Specific the Qemu machine type.",
 409         type => 'string',
 410         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 411         maxLength => 40,
 412         optional => 1,
 413     },
 414     smbios1 => {
 415         description => "Specify SMBIOS type 1 fields.",
 416         type => 'string', format => 'pve-qm-smbios1',
 417         maxLength => 256,
 418         optional => 1,
 419     },
 420     protection => {
 421         optional => 1,
 422         type => 'boolean',
 423         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 424         default => 0,
 425     },
 426     bios => {
 427         optional => 1,
 428         type => 'string',
 429         enum => [ qw(seabios ovmf) ],
 430         description => "Select BIOS implementation.",
 431         default => 'seabios',
 432     },
 433 };
 434
 435 # what about other qemu settings ?
 436 #cpu => 'string',
 437 #machine => 'string',
 438 #fda => 'file',
 439 #fdb => 'file',
 440 #mtdblock => 'file',
 441 #sd => 'file',
 442 #pflash => 'file',
 443 #snapshot => 'bool',
 444 #bootp => 'file',
 445 ##tftp => 'dir',
 446 ##smb => 'dir',
 447 #kernel => 'file',
 448 #append => 'string',
 449 #initrd => 'file',
 450 ##soundhw => 'string',
 451
 452 while (my ($k, $v) = each %$confdesc) {
 453     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 454 }
 455
 456 my $MAX_IDE_DISKS = 4;
 457 my $MAX_SCSI_DISKS = 14;
 458 my $MAX_VIRTIO_DISKS = 16;
 459 my $MAX_SATA_DISKS = 6;
 460 my $MAX_USB_DEVICES = 5;
 461 my $MAX_NETS = 32;
 462 my $MAX_UNUSED_DISKS = 8;
 463 my $MAX_HOSTPCI_DEVICES = 4;
 464 my $MAX_SERIAL_PORTS = 4;
 465 my $MAX_PARALLEL_PORTS = 3;
 466 my $MAX_NUMA = 8;
 467 my $MAX_MEM = 4194304;
 468 my $STATICMEM = 1024;
 469
 470 my $numa_fmt = {
 471     cpus => {
 472         type => "string",
 473         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 474         description => "CPUs accessing this numa node.",
 475         format_description => "id[-id];...",
 476     },
 477     memory => {
 478         type => "number",
 479         description => "Amount of memory this numa node provides.",
 480         format_description => "mb",
 481         optional => 1,
 482     },
 483     hostnodes => {
 484         type => "string",
 485         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 486         description => "host numa nodes to use",
 487         format_description => "id[-id];...",
 488         optional => 1,
 489     },
 490     policy => {
 491         type => 'string',
 492         enum => [qw(preferred bind interleave)],
 493         format_description => 'preferred|bind|interleave',
 494         description => "numa allocation policy.",
 495         optional => 1,
 496     },
 497 };
 498 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 499 my $numadesc = {
 500     optional => 1,
 501     type => 'string', format => $numa_fmt,
 502     description => "numa topology",
 503 };
 504 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 505
 506 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 507     $confdesc->{"numa$i"} = $numadesc;
 508 }
 509
 510 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 511                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 512                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 513 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 514
 515 my $net_fmt = {
 516     macaddr => {
 517         type => 'string',
 518         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 519         description => "MAC address",
 520         format_description => "XX:XX:XX:XX:XX:XX",
 521         optional => 1,
 522     },
 523     model => { alias => 'macaddr', default_key => 1 },
 524     (map { $_ => { group => 'model' } } @$nic_model_list),
 525     bridge => {
 526         type => 'string',
 527         description => 'Bridge to attach the network device to.',
 528         format_description => 'bridge',
 529         optional => 1,
 530     },
 531     queues => {
 532         type => 'integer',
 533         minimum => 0, maximum => 16,
 534         description => 'Number of packet queues to be used on the device.',
 535         format_description => 'number',
 536         optional => 1,
 537     },
 538     rate => {
 539         type => 'number',
 540         minimum => 0,
 541         description => 'Rate limit in mbps as floating point number.',
 542         format_description => 'mbps',
 543         optional => 1,
 544     },
 545     tag => {
 546         type => 'integer',
 547         minimum => 2, maximum => 4094,
 548         description => 'VLAN tag to apply to packets on this interface.',
 549         format_description => 'vlanid',
 550         optional => 1,
 551     },
 552     trunks => {
 553         type => 'string',
 554         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 555         description => 'VLAN trunks to pass through this interface.',
 556         format_description => 'id;id...',
 557         optional => 1,
 558     },
 559     firewall => {
 560         type => 'boolean',
 561         description => 'Whether this interface should be protected by the firewall.',
 562         format_description => '0|1',
 563         optional => 1,
 564     },
 565     link_down => {
 566         type => 'boolean',
 567         description => 'Whether this interface should be DISconnected (like pulling the plug).',
 568         format_description => '0|1',
 569         optional => 1,
 570     },
 571 };
 572 my $netdesc = {
 573     optional => 1,
 574     type => 'string', format => 'pve-qm-net',
 575     description => <<EODESCR,
 576 Specify network devices.
 577
 578 MODEL is one of: $nic_model_list_txt
 579
 580 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 581 automatically generated if not specified.
 582
 583 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 584
 585 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 586
 587 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 588
 589 10.0.2.2   Gateway
 590 10.0.2.3   DNS Server
 591 10.0.2.4   SMB Server
 592
 593 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 594
 595 EODESCR
 596 };
 597 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 598
 599 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 600     $confdesc->{"net$i"} = $netdesc;
 601 }
 602
 603 PVE::JSONSchema::register_format('pve-volume-id-or-none', \&verify_volume_id_or_none);
 604 sub verify_volume_id_or_none {
 605     my ($volid, $noerr) = @_;
 606
 607     return $volid if $volid eq 'none';
 608     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 609     if ($@) {
 610         return undef if $noerr;
 611         die $@;
 612     }
 613     return $volid;
 614 }
 615
 616 my $drivename_hash;
 617
 618 my %drivedesc_base = (
 619     volume => { alias => 'file' },
 620     file => {
 621         type => 'string',
 622         format => 'pve-volume-id-or-none',
 623         default_key => 1,
 624         format_description => 'volume',
 625         description => "The drive's backing volume.",
 626     },
 627     media => {
 628         type => 'string',
 629         format_description => 'cdrom|disk',
 630         enum => [qw(cdrom disk)],
 631         description => "The drive's media type.",
 632         default => 'disk',
 633         optional => 1
 634     },
 635     cyls => {
 636         type => 'integer',
 637         format_description => 'count',
 638         description => "Force the drive's physical geometry to have a specific cylinder count.",
 639         optional => 1
 640     },
 641     heads => {
 642         type => 'integer',
 643         format_description => 'count',
 644         description => "Force the drive's physical geometry to have a specific head count.",
 645         optional => 1
 646     },
 647     secs => {
 648         type => 'integer',
 649         format_description => 'count',
 650         description => "Force the drive's physical geometry to have a specific sector count.",
 651         optional => 1
 652     },
 653     trans => {
 654         type => 'string',
 655         format_description => 'none|lba|auto',
 656         enum => [qw(none lba auto)],
 657         description => "Force disk geometry bios translation mode.",
 658         optional => 1,
 659     },
 660     snapshot => {
 661         type => 'boolean',
 662         format_description => 'on|off',
 663         description => "Whether the drive should be included when making snapshots.",
 664         optional => 1,
 665     },
 666     cache => {
 667         type => 'string',
 668         format_description => 'none|writethrough|writeback|unsafe|directsync',
 669         enum => [qw(none writethrough writeback unsafe directsync)],
 670         description => "The drive's cache mode",
 671         optional => 1,
 672     },
 673     format => {
 674         type => 'string',
 675         format_description => 'drive format',
 676         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 677         description => "The drive's backing file's data format.",
 678         optional => 1,
 679     },
 680     size => {
 681         type => 'string',
 682         format => 'disk-size',
 683         description => "Disk size. This is purely informational and has no effect.",
 684         optional => 1,
 685     },
 686     backup => {
 687         type => 'boolean',
 688         format_description => 'on|off',
 689         description => "Whether the drive should be included when making backups.",
 690         optional => 1,
 691     },
 692     werror => {
 693         type => 'string',
 694         format_description => 'enospc|ignore|report|stop',
 695         enum => [qw(enospc ignore report stop)],
 696         description => 'Write error action.',
 697         optional => 1,
 698     },
 699     aio => {
 700         type => 'string',
 701         format_description => 'native|threads',
 702         enum => [qw(native threads)],
 703         description => 'AIO type to use.',
 704         optional => 1,
 705     },
 706     discard => {
 707         type => 'string',
 708         format_description => 'ignore|on',
 709         enum => [qw(ignore on)],
 710         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 711         optional => 1,
 712     },
 713     detect_zeroes => {
 714         type => 'boolean',
 715         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 716         optional => 1,
 717     },
 718     serial => {
 719         type => 'string',
 720         format => 'urlencoded',
 721         format_description => 'serial',
 722         maxLength => 20*3, # *3 since it's %xx url enoded
 723         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 724         optional => 1,
 725     }
 726 );
 727
 728 my %rerror_fmt = (
 729     rerror => {
 730         type => 'string',
 731         format_description => 'ignore|report|stop',
 732         enum => [qw(ignore report stop)],
 733         description => 'Read error action.',
 734         optional => 1,
 735     },
 736 );
 737
 738 my %iothread_fmt = ( iothread => {
 739         type => 'boolean',
 740         format_description => 'off|on',
 741         description => "Whether to use iothreads for this drive",
 742         optional => 1,
 743 });
 744
 745 my %model_fmt = (
 746     model => {
 747         type => 'string',
 748         format => 'urlencoded',
 749         format_description => 'model',
 750         maxLength => 40*3, # *3 since it's %xx url enoded
 751         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 752         optional => 1,
 753     },
 754 );
 755
 756 my %queues_fmt = (
 757     queues => {
 758         type => 'integer',
 759         format_description => 'nbqueues',
 760         description => "Number of queues.",
 761         minimum => 2,
 762         optional => 1
 763     }
 764 );
 765
 766 my $add_throttle_desc = sub {
 767     my ($key, $type, $what, $size, $longsize) = @_;
 768     $drivedesc_base{$key} = {
 769         type => $type,
 770         format_description => $size,
 771         description => "Maximum $what speed in $longsize per second.",
 772         optional => 1,
 773     };
 774 };
 775 # throughput: (leaky bucket)
 776 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 777 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 778 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 779 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 780 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 781 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 782 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 783 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 784 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 785
 786 # pools: (pool of IO before throttling starts taking effect)
 787 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 788 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 789 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 790 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 791 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 792 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 793
 794 my $ide_fmt = {
 795     %drivedesc_base,
 796     %rerror_fmt,
 797     %model_fmt,
 798 };
 799
 800 my $idedesc = {
 801     optional => 1,
 802     type => 'string', format => $ide_fmt,
 803     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 804 };
 805 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 806
 807 my $scsi_fmt = {
 808     %drivedesc_base,
 809     %iothread_fmt,
 810     %queues_fmt,
 811 };
 812 my $scsidesc = {
 813     optional => 1,
 814     type => 'string', format => $scsi_fmt,
 815     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 816 };
 817 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 818
 819 my $sata_fmt = {
 820     %drivedesc_base,
 821     %rerror_fmt,
 822 };
 823 my $satadesc = {
 824     optional => 1,
 825     type => 'string', format => $sata_fmt,
 826     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 827 };
 828 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 829
 830 my $virtio_fmt = {
 831     %drivedesc_base,
 832     %iothread_fmt,
 833     %rerror_fmt,
 834 };
 835 my $virtiodesc = {
 836     optional => 1,
 837     type => 'string', format => $virtio_fmt,
 838     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 839 };
 840 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 841
 842 my $alldrive_fmt = {
 843     %drivedesc_base,
 844     %rerror_fmt,
 845     %iothread_fmt,
 846     %model_fmt,
 847     %queues_fmt,
 848 };
 849
 850 my $usb_fmt = {
 851     host => {
 852         default_key => 1,
 853         type => 'string', format => 'pve-qm-usb-device',
 854         format_description => 'HOSTUSBDEVICE|spice',
 855         description => 'The Host USB device or port or the value spice',
 856     },
 857     usb3 => {
 858         optional => 1,
 859         type => 'boolean',
 860         format_description => 'yes|no',
 861         description => 'Specifies whether if given host option is a USB3 device or port',
 862     },
 863 };
 864
 865 my $usbdesc = {
 866     optional => 1,
 867     type => 'string', format => $usb_fmt,
 868     description => <<EODESCR,
 869 Configure an USB device (n is 0 to 4). This can be used to
 870 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 871
 872 'bus-port(.port)*' (decimal numbers) or
 873 'vendor_id:product_id' (hexadeciaml numbers) or
 874 'spice'
 875
 876 You can use the 'lsusb -t' command to list existing usb devices.
 877
 878 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 879
 880 The value 'spice' can be used to add a usb redirection devices for spice.
 881
 882 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 883
 884 EODESCR
 885 };
 886 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 887
 888 # NOTE: the match-groups of this regex are used in parse_hostpci
 889 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 890 my $hostpci_fmt = {
 891     host => {
 892         default_key => 1,
 893         type => 'string',
 894         pattern => qr/$PCIRE(;$PCIRE)*/,
 895         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 896         description => "The PCI ID of a host's PCI device or a list of PCI virtual functions of the host.",
 897     },
 898     rombar => {
 899         type => 'boolean',
 900         optional => 1,
 901         default => 1,
 902     },
 903     pcie => {
 904         type => 'boolean',
 905         optional => 1,
 906         default => 0,
 907     },
 908     'x-vga' => {
 909         type => 'boolean',
 910         optional => 1,
 911         default => 0,
 912     },
 913 };
 914 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
 915
 916 my $hostpcidesc = {
 917         optional => 1,
 918         type => 'string', format => 'pve-qm-hostpci',
 919         description => <<EODESCR,
 920 Map host pci devices. HOSTPCIDEVICE syntax is:
 921
 922 'bus:dev.func' (hexadecimal numbers)
 923
 924 You can us the 'lspci' command to list existing pci devices.
 925
 926 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 927
 928 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 929
 930 Experimental: user reported problems with this option.
 931 EODESCR
 932 };
 933 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 934
 935 my $serialdesc = {
 936         optional => 1,
 937         type => 'string',
 938         pattern => '(/dev/.+|socket)',
 939         description =>  <<EODESCR,
 940 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 941
 942 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 943
 944 Experimental: user reported problems with this option.
 945 EODESCR
 946 };
 947
 948 my $paralleldesc= {
 949         optional => 1,
 950         type => 'string',
 951         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 952         description =>  <<EODESCR,
 953 Map host parallel devices (n is 0 to 2).
 954
 955 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 956
 957 Experimental: user reported problems with this option.
 958 EODESCR
 959 };
 960
 961 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 962     $confdesc->{"parallel$i"} = $paralleldesc;
 963 }
 964
 965 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 966     $confdesc->{"serial$i"} = $serialdesc;
 967 }
 968
 969 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 970     $confdesc->{"hostpci$i"} = $hostpcidesc;
 971 }
 972
 973 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 974     $drivename_hash->{"ide$i"} = 1;
 975     $confdesc->{"ide$i"} = $idedesc;
 976 }
 977
 978 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 979     $drivename_hash->{"sata$i"} = 1;
 980     $confdesc->{"sata$i"} = $satadesc;
 981 }
 982
 983 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 984     $drivename_hash->{"scsi$i"} = 1;
 985     $confdesc->{"scsi$i"} = $scsidesc ;
 986 }
 987
 988 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 989     $drivename_hash->{"virtio$i"} = 1;
 990     $confdesc->{"virtio$i"} = $virtiodesc;
 991 }
 992
 993 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 994     $confdesc->{"usb$i"} = $usbdesc;
 995 }
 996
 997 my $unuseddesc = {
 998     optional => 1,
 999     type => 'string', format => 'pve-volume-id',
1000     description => "Reference to unused volumes.",
1001 };
1002
1003 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1004     $confdesc->{"unused$i"} = $unuseddesc;
1005 }
1006
1007 my $kvm_api_version = 0;
1008
1009 sub kvm_version {
1010
1011     return $kvm_api_version if $kvm_api_version;
1012
1013     my $fh = IO::File->new("</dev/kvm") ||
1014         return 0;
1015
1016     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1017         $kvm_api_version = $v;
1018     }
1019
1020     $fh->close();
1021
1022     return  $kvm_api_version;
1023 }
1024
1025 my $kvm_user_version;
1026
1027 sub kvm_user_version {
1028
1029     return $kvm_user_version if $kvm_user_version;
1030
1031     $kvm_user_version = 'unknown';
1032
1033     my $code = sub {
1034         my $line = shift;
1035         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1036             $kvm_user_version = $2;
1037         }
1038     };
1039
1040     eval { run_command("kvm -version", outfunc => $code); };
1041     warn $@ if $@;
1042
1043     return $kvm_user_version;
1044
1045 }
1046
1047 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1048
1049 sub valid_drive_names {
1050     # order is important - used to autoselect boot disk
1051     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1052             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1053             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1054             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1055 }
1056
1057 sub is_valid_drivename {
1058     my $dev = shift;
1059
1060     return defined($drivename_hash->{$dev});
1061 }
1062
1063 sub option_exists {
1064     my $key = shift;
1065     return defined($confdesc->{$key});
1066 }
1067
1068 sub nic_models {
1069     return $nic_model_list;
1070 }
1071
1072 sub os_list_description {
1073
1074     return {
1075         other => 'Other',
1076         wxp => 'Windows XP',
1077         w2k => 'Windows 2000',
1078         w2k3 =>, 'Windows 2003',
1079         w2k8 => 'Windows 2008',
1080         wvista => 'Windows Vista',
1081         win7 => 'Windows 7',
1082         win8 => 'Windows 8/2012',
1083         l24 => 'Linux 2.4',
1084         l26 => 'Linux 2.6',
1085     };
1086 }
1087
1088 my $cdrom_path;
1089
1090 sub get_cdrom_path {
1091
1092     return  $cdrom_path if $cdrom_path;
1093
1094     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1095     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1096     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1097 }
1098
1099 sub get_iso_path {
1100     my ($storecfg, $vmid, $cdrom) = @_;
1101
1102     if ($cdrom eq 'cdrom') {
1103         return get_cdrom_path();
1104     } elsif ($cdrom eq 'none') {
1105         return '';
1106     } elsif ($cdrom =~ m|^/|) {
1107         return $cdrom;
1108     } else {
1109         return PVE::Storage::path($storecfg, $cdrom);
1110     }
1111 }
1112
1113 # try to convert old style file names to volume IDs
1114 sub filename_to_volume_id {
1115     my ($vmid, $file, $media) = @_;
1116
1117     if (!($file eq 'none' || $file eq 'cdrom' ||
1118           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1119
1120         return undef if $file =~ m|/|;
1121
1122         if ($media && $media eq 'cdrom') {
1123             $file = "local:iso/$file";
1124         } else {
1125             $file = "local:$vmid/$file";
1126         }
1127     }
1128
1129     return $file;
1130 }
1131
1132 sub verify_media_type {
1133     my ($opt, $vtype, $media) = @_;
1134
1135     return if !$media;
1136
1137     my $etype;
1138     if ($media eq 'disk') {
1139         $etype = 'images';
1140     } elsif ($media eq 'cdrom') {
1141         $etype = 'iso';
1142     } else {
1143         die "internal error";
1144     }
1145
1146     return if ($vtype eq $etype);
1147
1148     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1149 }
1150
1151 sub cleanup_drive_path {
1152     my ($opt, $storecfg, $drive) = @_;
1153
1154     # try to convert filesystem paths to volume IDs
1155
1156     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1157         ($drive->{file} !~ m|^/dev/.+|) &&
1158         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1159         ($drive->{file} !~ m/^\d+$/)) {
1160         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1161         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1162         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1163         verify_media_type($opt, $vtype, $drive->{media});
1164         $drive->{file} = $volid;
1165     }
1166
1167     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1168 }
1169
1170 sub parse_hotplug_features {
1171     my ($data) = @_;
1172
1173     my $res = {};
1174
1175     return $res if $data eq '0';
1176
1177     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1178
1179     foreach my $feature (PVE::Tools::split_list($data)) {
1180         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1181             $res->{$1} = 1;
1182         } else {
1183             warn "ignoring unknown hotplug feature '$feature'\n";
1184         }
1185     }
1186     return $res;
1187 }
1188
1189 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1190 sub pve_verify_hotplug_features {
1191     my ($value, $noerr) = @_;
1192
1193     return $value if parse_hotplug_features($value);
1194
1195     return undef if $noerr;
1196
1197     die "unable to parse hotplug option\n";
1198 }
1199
1200 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1201 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1202 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1203 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1204 #        [,iothread=on][,serial=serial][,model=model]
1205
1206 sub parse_drive {
1207     my ($key, $data) = @_;
1208
1209     my ($interface, $index);
1210
1211     if ($key =~ m/^([^\d]+)(\d+)$/) {
1212         $interface = $1;
1213         $index = $2;
1214     } else {
1215         return undef;
1216     }
1217
1218     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1219                                      : $confdesc->{$key}->{format};
1220     if (!$desc) {
1221         warn "invalid drive key: $key\n";
1222         return undef;
1223     }
1224     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1225     return undef if !$res;
1226     $res->{interface} = $interface;
1227     $res->{index} = $index;
1228
1229     my $error = 0;
1230     foreach my $opt (qw(bps bps_rd bps_wr)) {
1231         if (my $bps = defined(delete $res->{$opt})) {
1232             if (defined($res->{"m$opt"})) {
1233                 warn "both $opt and m$opt specified\n";
1234                 ++$error;
1235                 next;
1236             }
1237             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1238         }
1239     }
1240     return undef if $error;
1241
1242     return undef if $res->{mbps_rd} && $res->{mbps};
1243     return undef if $res->{mbps_wr} && $res->{mbps};
1244     return undef if $res->{iops_rd} && $res->{iops};
1245     return undef if $res->{iops_wr} && $res->{iops};
1246
1247     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1248         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1249         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1250         return undef if $res->{interface} eq 'virtio';
1251     }
1252
1253     if (my $size = $res->{size}) {
1254         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1255     }
1256
1257     return $res;
1258 }
1259
1260 sub print_drive {
1261     my ($vmid, $drive) = @_;
1262     my $data = { %$drive };
1263     delete $data->{$_} for qw(index interface);
1264     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1265 }
1266
1267 sub scsi_inquiry {
1268     my($fh, $noerr) = @_;
1269
1270     my $SG_IO = 0x2285;
1271     my $SG_GET_VERSION_NUM = 0x2282;
1272
1273     my $versionbuf = "\x00" x 8;
1274     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1275     if (!$ret) {
1276         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1277         return undef;
1278     }
1279     my $version = unpack("I", $versionbuf);
1280     if ($version < 30000) {
1281         die "scsi generic interface too old\n"  if !$noerr;
1282         return undef;
1283     }
1284
1285     my $buf = "\x00" x 36;
1286     my $sensebuf = "\x00" x 8;
1287     my $cmd = pack("C x3 C x1", 0x12, 36);
1288
1289     # see /usr/include/scsi/sg.h
1290     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1291
1292     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1293                       length($sensebuf), 0, length($buf), $buf,
1294                       $cmd, $sensebuf, 6000);
1295
1296     $ret = ioctl($fh, $SG_IO, $packet);
1297     if (!$ret) {
1298         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1299         return undef;
1300     }
1301
1302     my @res = unpack($sg_io_hdr_t, $packet);
1303     if ($res[17] || $res[18]) {
1304         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1305         return undef;
1306     }
1307
1308     my $res = {};
1309     (my $byte0, my $byte1, $res->{vendor},
1310      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1311
1312     $res->{removable} = $byte1 & 128 ? 1 : 0;
1313     $res->{type} = $byte0 & 31;
1314
1315     return $res;
1316 }
1317
1318 sub path_is_scsi {
1319     my ($path) = @_;
1320
1321     my $fh = IO::File->new("+<$path") || return undef;
1322     my $res = scsi_inquiry($fh, 1);
1323     close($fh);
1324
1325     return $res;
1326 }
1327
1328 sub machine_type_is_q35 {
1329     my ($conf) = @_;
1330
1331     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1332 }
1333
1334 sub print_tabletdevice_full {
1335     my ($conf) = @_;
1336
1337     my $q35 = machine_type_is_q35($conf);
1338
1339     # we use uhci for old VMs because tablet driver was buggy in older qemu
1340     my $usbbus = $q35 ? "ehci" : "uhci";
1341
1342     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1343 }
1344
1345 sub print_drivedevice_full {
1346     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1347
1348     my $device = '';
1349     my $maxdev = 0;
1350
1351     if ($drive->{interface} eq 'virtio') {
1352         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1353         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1354         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1355     } elsif ($drive->{interface} eq 'scsi') {
1356
1357         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1358         my $unit = $drive->{index} % $maxdev;
1359         my $devicetype = 'hd';
1360         my $path = '';
1361         if (drive_is_cdrom($drive)) {
1362             $devicetype = 'cd';
1363         } else {
1364             if ($drive->{file} =~ m|^/|) {
1365                 $path = $drive->{file};
1366                 if (my $info = path_is_scsi($path)) {
1367                     if ($info->{type} == 0) {
1368                         $devicetype = 'block';
1369                     } elsif ($info->{type} == 1) { # tape
1370                         $devicetype = 'generic';
1371                     }
1372                 }
1373             } else {
1374                  $path = PVE::Storage::path($storecfg, $drive->{file});
1375             }
1376
1377             if($path =~ m/^iscsi\:\/\//){
1378                 $devicetype = 'generic';
1379             }
1380         }
1381
1382         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1383             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1384         } else {
1385             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1386         }
1387
1388     } elsif ($drive->{interface} eq 'ide'){
1389         $maxdev = 2;
1390         my $controller = int($drive->{index} / $maxdev);
1391         my $unit = $drive->{index} % $maxdev;
1392         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1393
1394         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1395         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1396             $model = URI::Escape::uri_unescape($model);
1397             $device .= ",model=$model";
1398         }
1399     } elsif ($drive->{interface} eq 'sata'){
1400         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1401         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1402         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1403     } elsif ($drive->{interface} eq 'usb') {
1404         die "implement me";
1405         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1406     } else {
1407         die "unsupported interface type";
1408     }
1409
1410     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1411
1412     return $device;
1413 }
1414
1415 sub get_initiator_name {
1416     my $initiator;
1417
1418     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1419     while (defined(my $line = <$fh>)) {
1420         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1421         $initiator = $1;
1422         last;
1423     }
1424     $fh->close();
1425
1426     return $initiator;
1427 }
1428
1429 sub print_drive_full {
1430     my ($storecfg, $vmid, $drive) = @_;
1431
1432     my $path;
1433     my $volid = $drive->{file};
1434     my $format;
1435
1436     if (drive_is_cdrom($drive)) {
1437         $path = get_iso_path($storecfg, $vmid, $volid);
1438     } else {
1439         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1440         if ($storeid) {
1441             $path = PVE::Storage::path($storecfg, $volid);
1442             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1443             $format = qemu_img_format($scfg, $volname);
1444         } else {
1445             $path = $volid;
1446             $format = "raw";
1447         }
1448    }
1449
1450     my $opts = '';
1451     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1452     foreach my $o (@qemu_drive_options) {
1453         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1454     }
1455     if (my $serial = $drive->{serial}) {
1456         $serial = URI::Escape::uri_unescape($serial);
1457         $opts .= ",serial=$serial";
1458     }
1459
1460     $opts .= ",format=$format" if $format && !$drive->{format};
1461
1462     foreach my $o (qw(bps bps_rd bps_wr)) {
1463         my $v = $drive->{"m$o"};
1464         $opts .= ",$o=" . int($v*1024*1024) if $v;
1465     }
1466
1467     my $cache_direct = 0;
1468
1469     if (my $cache = $drive->{cache}) {
1470         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1471     } elsif (!drive_is_cdrom($drive)) {
1472         $opts .= ",cache=none";
1473         $cache_direct = 1;
1474     }
1475
1476     # aio native works only with O_DIRECT
1477     if (!$drive->{aio}) {
1478         if($cache_direct) {
1479             $opts .= ",aio=native";
1480         } else {
1481             $opts .= ",aio=threads";
1482         }
1483     }
1484
1485     if (!drive_is_cdrom($drive)) {
1486         my $detectzeroes;
1487         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1488             $detectzeroes = 'off';
1489         } elsif ($drive->{discard}) {
1490             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1491         } else {
1492             # This used to be our default with discard not being specified:
1493             $detectzeroes = 'on';
1494         }
1495         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1496     }
1497
1498     my $pathinfo = $path ? "file=$path," : '';
1499
1500     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1501 }
1502
1503 sub print_netdevice_full {
1504     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1505
1506     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1507
1508     my $device = $net->{model};
1509     if ($net->{model} eq 'virtio') {
1510          $device = 'virtio-net-pci';
1511      };
1512
1513     my $pciaddr = print_pci_addr("$netid", $bridges);
1514     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1515     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1516         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1517         my $vectors = $net->{queues} * 2 + 2;
1518         $tmpstr .= ",vectors=$vectors,mq=on";
1519     }
1520     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1521
1522     if ($use_old_bios_files) {
1523         my $romfile;
1524         if ($device eq 'virtio-net-pci') {
1525             $romfile = 'pxe-virtio.rom';
1526         } elsif ($device eq 'e1000') {
1527             $romfile = 'pxe-e1000.rom';
1528         } elsif ($device eq 'ne2k') {
1529             $romfile = 'pxe-ne2k_pci.rom';
1530         } elsif ($device eq 'pcnet') {
1531             $romfile = 'pxe-pcnet.rom';
1532         } elsif ($device eq 'rtl8139') {
1533             $romfile = 'pxe-rtl8139.rom';
1534         }
1535         $tmpstr .= ",romfile=$romfile" if $romfile;
1536     }
1537
1538     return $tmpstr;
1539 }
1540
1541 sub print_netdev_full {
1542     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1543
1544     my $i = '';
1545     if ($netid =~ m/^net(\d+)$/) {
1546         $i = int($1);
1547     }
1548
1549     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1550
1551     my $ifname = "tap${vmid}i$i";
1552
1553     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1554     die "interface name '$ifname' is too long (max 15 character)\n"
1555         if length($ifname) >= 16;
1556
1557     my $vhostparam = '';
1558     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1559
1560     my $vmname = $conf->{name} || "vm$vmid";
1561
1562     my $netdev = "";
1563     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1564
1565     if ($net->{bridge}) {
1566         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1567     } else {
1568         $netdev = "type=user,id=$netid,hostname=$vmname";
1569     }
1570
1571     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1572
1573     return $netdev;
1574 }
1575
1576 sub drive_is_cdrom {
1577     my ($drive) = @_;
1578
1579     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1580
1581 }
1582
1583 sub parse_number_sets {
1584     my ($set) = @_;
1585     my $res = [];
1586     foreach my $part (split(/;/, $set)) {
1587         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1588             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1589             push @$res, [ $1, $2 ];
1590         } else {
1591             die "invalid range: $part\n";
1592         }
1593     }
1594     return $res;
1595 }
1596
1597 sub parse_numa {
1598     my ($data) = @_;
1599
1600     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1601     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1602     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1603     return $res;
1604 }
1605
1606 sub parse_hostpci {
1607     my ($value) = @_;
1608
1609     return undef if !$value;
1610
1611     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1612
1613     my @idlist = split(/;/, $res->{host});
1614     delete $res->{host};
1615     foreach my $id (@idlist) {
1616         if ($id =~ /^$PCIRE$/) {
1617             push @{$res->{pciid}}, { id => $1, function => ($2//'0') };
1618         } else {
1619             # should have been caught by parse_property_string already
1620             die "failed to parse PCI id: $id\n";
1621         }
1622     }
1623     return $res;
1624 }
1625
1626 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1627 sub parse_net {
1628     my ($data) = @_;
1629
1630     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1631     if ($@) {
1632         warn $@;
1633         return undef;
1634     }
1635     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1636     return $res;
1637 }
1638
1639 sub print_net {
1640     my $net = shift;
1641
1642     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1643 }
1644
1645 sub add_random_macs {
1646     my ($settings) = @_;
1647
1648     foreach my $opt (keys %$settings) {
1649         next if $opt !~ m/^net(\d+)$/;
1650         my $net = parse_net($settings->{$opt});
1651         next if !$net;
1652         $settings->{$opt} = print_net($net);
1653     }
1654 }
1655
1656 sub vm_is_volid_owner {
1657     my ($storecfg, $vmid, $volid) = @_;
1658
1659     if ($volid !~  m|^/|) {
1660         my ($path, $owner);
1661         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1662         if ($owner && ($owner == $vmid)) {
1663             return 1;
1664         }
1665     }
1666
1667     return undef;
1668 }
1669
1670 sub split_flagged_list {
1671     my $text = shift || '';
1672     $text =~ s/[,;]/ /g;
1673     $text =~ s/^\s+//;
1674     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1675 }
1676
1677 sub join_flagged_list {
1678     my ($how, $lst) = @_;
1679     join $how, map { $lst->{$_} . $_ } keys %$lst;
1680 }
1681
1682 sub vmconfig_delete_pending_option {
1683     my ($conf, $key, $force) = @_;
1684
1685     delete $conf->{pending}->{$key};
1686     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1687     $pending_delete_hash->{$key} = $force ? '!' : '';
1688     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1689 }
1690
1691 sub vmconfig_undelete_pending_option {
1692     my ($conf, $key) = @_;
1693
1694     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1695     delete $pending_delete_hash->{$key};
1696
1697     if (%$pending_delete_hash) {
1698         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1699     } else {
1700         delete $conf->{pending}->{delete};
1701     }
1702 }
1703
1704 sub vmconfig_register_unused_drive {
1705     my ($storecfg, $vmid, $conf, $drive) = @_;
1706
1707     if (!drive_is_cdrom($drive)) {
1708         my $volid = $drive->{file};
1709         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1710             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1711         }
1712     }
1713 }
1714
1715 sub vmconfig_cleanup_pending {
1716     my ($conf) = @_;
1717
1718     # remove pending changes when nothing changed
1719     my $changes;
1720     foreach my $opt (keys %{$conf->{pending}}) {
1721         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1722             $changes = 1;
1723             delete $conf->{pending}->{$opt};
1724         }
1725     }
1726
1727     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1728     my $pending_delete_hash = {};
1729     while (my ($opt, $force) = each %$current_delete_hash) {
1730         if (defined($conf->{$opt})) {
1731             $pending_delete_hash->{$opt} = $force;
1732         } else {
1733             $changes = 1;
1734         }
1735     }
1736
1737     if (%$pending_delete_hash) {
1738         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1739     } else {
1740         delete $conf->{pending}->{delete};
1741     }
1742
1743     return $changes;
1744 }
1745
1746 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1747 my $smbios1_fmt = {
1748     uuid => {
1749         type => 'string',
1750         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1751         format_description => 'UUID',
1752         optional => 1,
1753     },
1754     version => {
1755         type => 'string',
1756         pattern => '\S+',
1757         format_description => 'str',
1758         optional => 1,
1759     },
1760     serial => {
1761         type => 'string',
1762         pattern => '\S+',
1763         format_description => 'str',
1764         optional => 1,
1765     },
1766     manufacturer => {
1767         type => 'string',
1768         pattern => '\S+',
1769         format_description => 'name',
1770         optional => 1,
1771     },
1772     product => {
1773         type => 'string',
1774         pattern => '\S+',
1775         format_description => 'name',
1776         optional => 1,
1777     },
1778     sku => {
1779         type => 'string',
1780         pattern => '\S+',
1781         format_description => 'str',
1782         optional => 1,
1783     },
1784     family => {
1785         type => 'string',
1786         pattern => '\S+',
1787         format_description => 'str',
1788         optional => 1,
1789     },
1790 };
1791
1792 sub parse_smbios1 {
1793     my ($data) = @_;
1794
1795     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1796     warn $@ if $@;
1797     return $res;
1798 }
1799
1800 sub print_smbios1 {
1801     my ($smbios1) = @_;
1802     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1803 }
1804
1805 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1806
1807 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1808 sub verify_bootdisk {
1809     my ($value, $noerr) = @_;
1810
1811     return $value if is_valid_drivename($value);
1812
1813     return undef if $noerr;
1814
1815     die "invalid boot disk '$value'\n";
1816 }
1817
1818 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1819 sub verify_net {
1820     my ($value, $noerr) = @_;
1821
1822     return $value if parse_net($value);
1823
1824     return undef if $noerr;
1825
1826     die "unable to parse network options\n";
1827 }
1828
1829 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1830 sub verify_watchdog {
1831     my ($value, $noerr) = @_;
1832
1833     return $value if parse_watchdog($value);
1834
1835     return undef if $noerr;
1836
1837     die "unable to parse watchdog options\n";
1838 }
1839
1840 sub parse_watchdog {
1841     my ($value) = @_;
1842
1843     return undef if !$value;
1844
1845     my $res = {};
1846
1847     foreach my $p (split(/,/, $value)) {
1848         next if $p =~ m/^\s*$/;
1849
1850         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1851             $res->{model} = $2;
1852         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1853             $res->{action} = $2;
1854         } else {
1855             return undef;
1856         }
1857     }
1858
1859     return $res;
1860 }
1861
1862 sub parse_usb_device {
1863     my ($value) = @_;
1864
1865     return undef if !$value;
1866
1867     my $res = {};
1868     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1869         $res->{vendorid} = $2;
1870         $res->{productid} = $4;
1871     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1872         $res->{hostbus} = $1;
1873         $res->{hostport} = $2;
1874     } elsif ($value =~ m/^spice$/i) {
1875         $res->{spice} = 1;
1876     } else {
1877         return undef;
1878     }
1879
1880     return $res;
1881 }
1882
1883 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1884 sub verify_usb_device {
1885     my ($value, $noerr) = @_;
1886
1887     return $value if parse_usb_device($value);
1888
1889     return undef if $noerr;
1890
1891     die "unable to parse usb device\n";
1892 }
1893
1894 # add JSON properties for create and set function
1895 sub json_config_properties {
1896     my $prop = shift;
1897
1898     foreach my $opt (keys %$confdesc) {
1899         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1900         $prop->{$opt} = $confdesc->{$opt};
1901     }
1902
1903     return $prop;
1904 }
1905
1906 sub check_type {
1907     my ($key, $value) = @_;
1908
1909     die "unknown setting '$key'\n" if !$confdesc->{$key};
1910
1911     my $type = $confdesc->{$key}->{type};
1912
1913     if (!defined($value)) {
1914         die "got undefined value\n";
1915     }
1916
1917     if ($value =~ m/[\n\r]/) {
1918         die "property contains a line feed\n";
1919     }
1920
1921     if ($type eq 'boolean') {
1922         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1923         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1924         die "type check ('boolean') failed - got '$value'\n";
1925     } elsif ($type eq 'integer') {
1926         return int($1) if $value =~ m/^(\d+)$/;
1927         die "type check ('integer') failed - got '$value'\n";
1928     } elsif ($type eq 'number') {
1929         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1930         die "type check ('number') failed - got '$value'\n";
1931     } elsif ($type eq 'string') {
1932         if (my $fmt = $confdesc->{$key}->{format}) {
1933             if ($fmt eq 'pve-qm-drive') {
1934                 # special case - we need to pass $key to parse_drive()
1935                 my $drive = parse_drive($key, $value);
1936                 return $value if $drive;
1937                 die "unable to parse drive options\n";
1938             }
1939             PVE::JSONSchema::check_format($fmt, $value);
1940             return $value;
1941         }
1942         $value =~ s/^\"(.*)\"$/$1/;
1943         return $value;
1944     } else {
1945         die "internal error"
1946     }
1947 }
1948
1949 sub check_iommu_support{
1950     #fixme : need to check IOMMU support
1951     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1952
1953     my $iommu=1;
1954     return $iommu;
1955
1956 }
1957
1958 sub touch_config {
1959     my ($vmid) = @_;
1960
1961     my $conf = PVE::QemuConfig->config_file($vmid);
1962     utime undef, undef, $conf;
1963 }
1964
1965 sub destroy_vm {
1966     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1967
1968     my $conffile = PVE::QemuConfig->config_file($vmid);
1969
1970     my $conf = PVE::QemuConfig->load_config($vmid);
1971
1972     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1973
1974     # only remove disks owned by this VM
1975     foreach_drive($conf, sub {
1976         my ($ds, $drive) = @_;
1977
1978         return if drive_is_cdrom($drive);
1979
1980         my $volid = $drive->{file};
1981
1982         return if !$volid || $volid =~ m|^/|;
1983
1984         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1985         return if !$path || !$owner || ($owner != $vmid);
1986
1987         PVE::Storage::vdisk_free($storecfg, $volid);
1988     });
1989
1990     if ($keep_empty_config) {
1991         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1992     } else {
1993         unlink $conffile;
1994     }
1995
1996     # also remove unused disk
1997     eval {
1998         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1999
2000         eval {
2001             PVE::Storage::foreach_volid($dl, sub {
2002                 my ($volid, $sid, $volname, $d) = @_;
2003                 PVE::Storage::vdisk_free($storecfg, $volid);
2004             });
2005         };
2006         warn $@ if $@;
2007
2008     };
2009     warn $@ if $@;
2010 }
2011
2012 sub parse_vm_config {
2013     my ($filename, $raw) = @_;
2014
2015     return undef if !defined($raw);
2016
2017     my $res = {
2018         digest => Digest::SHA::sha1_hex($raw),
2019         snapshots => {},
2020         pending => {},
2021     };
2022
2023     $filename =~ m|/qemu-server/(\d+)\.conf$|
2024         || die "got strange filename '$filename'";
2025
2026     my $vmid = $1;
2027
2028     my $conf = $res;
2029     my $descr;
2030     my $section = '';
2031
2032     my @lines = split(/\n/, $raw);
2033     foreach my $line (@lines) {
2034         next if $line =~ m/^\s*$/;
2035
2036         if ($line =~ m/^\[PENDING\]\s*$/i) {
2037             $section = 'pending';
2038             if (defined($descr)) {
2039                 $descr =~ s/\s+$//;
2040                 $conf->{description} = $descr;
2041             }
2042             $descr = undef;
2043             $conf = $res->{$section} = {};
2044             next;
2045
2046         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2047             $section = $1;
2048             if (defined($descr)) {
2049                 $descr =~ s/\s+$//;
2050                 $conf->{description} = $descr;
2051             }
2052             $descr = undef;
2053             $conf = $res->{snapshots}->{$section} = {};
2054             next;
2055         }
2056
2057         if ($line =~ m/^\#(.*)\s*$/) {
2058             $descr = '' if !defined($descr);
2059             $descr .= PVE::Tools::decode_text($1) . "\n";
2060             next;
2061         }
2062
2063         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2064             $descr = '' if !defined($descr);
2065             $descr .= PVE::Tools::decode_text($2);
2066         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2067             $conf->{snapstate} = $1;
2068         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2069             my $key = $1;
2070             my $value = $2;
2071             $conf->{$key} = $value;
2072         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2073             my $value = $1;
2074             if ($section eq 'pending') {
2075                 $conf->{delete} = $value; # we parse this later
2076             } else {
2077                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2078             }
2079         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2080             my $key = $1;
2081             my $value = $2;
2082             eval { $value = check_type($key, $value); };
2083             if ($@) {
2084                 warn "vm $vmid - unable to parse value of '$key' - $@";
2085             } else {
2086                 my $fmt = $confdesc->{$key}->{format};
2087                 if ($fmt && $fmt eq 'pve-qm-drive') {
2088                     my $v = parse_drive($key, $value);
2089                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2090                         $v->{file} = $volid;
2091                         $value = print_drive($vmid, $v);
2092                     } else {
2093                         warn "vm $vmid - unable to parse value of '$key'\n";
2094                         next;
2095                     }
2096                 }
2097
2098                 if ($key eq 'cdrom') {
2099                     $conf->{ide2} = $value;
2100                 } else {
2101                     $conf->{$key} = $value;
2102                 }
2103             }
2104         }
2105     }
2106
2107     if (defined($descr)) {
2108         $descr =~ s/\s+$//;
2109         $conf->{description} = $descr;
2110     }
2111     delete $res->{snapstate}; # just to be sure
2112
2113     return $res;
2114 }
2115
2116 sub write_vm_config {
2117     my ($filename, $conf) = @_;
2118
2119     delete $conf->{snapstate}; # just to be sure
2120
2121     if ($conf->{cdrom}) {
2122         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2123         $conf->{ide2} = $conf->{cdrom};
2124         delete $conf->{cdrom};
2125     }
2126
2127     # we do not use 'smp' any longer
2128     if ($conf->{sockets}) {
2129         delete $conf->{smp};
2130     } elsif ($conf->{smp}) {
2131         $conf->{sockets} = $conf->{smp};
2132         delete $conf->{cores};
2133         delete $conf->{smp};
2134     }
2135
2136     my $used_volids = {};
2137
2138     my $cleanup_config = sub {
2139         my ($cref, $pending, $snapname) = @_;
2140
2141         foreach my $key (keys %$cref) {
2142             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2143                 $key eq 'snapstate' || $key eq 'pending';
2144             my $value = $cref->{$key};
2145             if ($key eq 'delete') {
2146                 die "propertry 'delete' is only allowed in [PENDING]\n"
2147                     if !$pending;
2148                 # fixme: check syntax?
2149                 next;
2150             }
2151             eval { $value = check_type($key, $value); };
2152             die "unable to parse value of '$key' - $@" if $@;
2153
2154             $cref->{$key} = $value;
2155
2156             if (!$snapname && is_valid_drivename($key)) {
2157                 my $drive = parse_drive($key, $value);
2158                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2159             }
2160         }
2161     };
2162
2163     &$cleanup_config($conf);
2164
2165     &$cleanup_config($conf->{pending}, 1);
2166
2167     foreach my $snapname (keys %{$conf->{snapshots}}) {
2168         die "internal error" if $snapname eq 'pending';
2169         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2170     }
2171
2172     # remove 'unusedX' settings if we re-add a volume
2173     foreach my $key (keys %$conf) {
2174         my $value = $conf->{$key};
2175         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2176             delete $conf->{$key};
2177         }
2178     }
2179
2180     my $generate_raw_config = sub {
2181         my ($conf, $pending) = @_;
2182
2183         my $raw = '';
2184
2185         # add description as comment to top of file
2186         if (defined(my $descr = $conf->{description})) {
2187             if ($descr) {
2188                 foreach my $cl (split(/\n/, $descr)) {
2189                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2190                 }
2191             } else {
2192                 $raw .= "#\n" if $pending;
2193             }
2194         }
2195
2196         foreach my $key (sort keys %$conf) {
2197             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2198             $raw .= "$key: $conf->{$key}\n";
2199         }
2200         return $raw;
2201     };
2202
2203     my $raw = &$generate_raw_config($conf);
2204
2205     if (scalar(keys %{$conf->{pending}})){
2206         $raw .= "\n[PENDING]\n";
2207         $raw .= &$generate_raw_config($conf->{pending}, 1);
2208     }
2209
2210     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2211         $raw .= "\n[$snapname]\n";
2212         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2213     }
2214
2215     return $raw;
2216 }
2217
2218 sub load_defaults {
2219
2220     my $res = {};
2221
2222     # we use static defaults from our JSON schema configuration
2223     foreach my $key (keys %$confdesc) {
2224         if (defined(my $default = $confdesc->{$key}->{default})) {
2225             $res->{$key} = $default;
2226         }
2227     }
2228
2229     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2230     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2231
2232     return $res;
2233 }
2234
2235 sub config_list {
2236     my $vmlist = PVE::Cluster::get_vmlist();
2237     my $res = {};
2238     return $res if !$vmlist || !$vmlist->{ids};
2239     my $ids = $vmlist->{ids};
2240
2241     foreach my $vmid (keys %$ids) {
2242         my $d = $ids->{$vmid};
2243         next if !$d->{node} || $d->{node} ne $nodename;
2244         next if !$d->{type} || $d->{type} ne 'qemu';
2245         $res->{$vmid}->{exists} = 1;
2246     }
2247     return $res;
2248 }
2249
2250 # test if VM uses local resources (to prevent migration)
2251 sub check_local_resources {
2252     my ($conf, $noerr) = @_;
2253
2254     my $loc_res = 0;
2255
2256     $loc_res = 1 if $conf->{hostusb}; # old syntax
2257     $loc_res = 1 if $conf->{hostpci}; # old syntax
2258
2259     foreach my $k (keys %$conf) {
2260         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2261         # sockets are safe: they will recreated be on the target side post-migrate
2262         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2263         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2264     }
2265
2266     die "VM uses local resources\n" if $loc_res && !$noerr;
2267
2268     return $loc_res;
2269 }
2270
2271 # check if used storages are available on all nodes (use by migrate)
2272 sub check_storage_availability {
2273     my ($storecfg, $conf, $node) = @_;
2274
2275     foreach_drive($conf, sub {
2276         my ($ds, $drive) = @_;
2277
2278         my $volid = $drive->{file};
2279         return if !$volid;
2280
2281         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2282         return if !$sid;
2283
2284         # check if storage is available on both nodes
2285         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2286         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2287    });
2288 }
2289
2290 # list nodes where all VM images are available (used by has_feature API)
2291 sub shared_nodes {
2292     my ($conf, $storecfg) = @_;
2293
2294     my $nodelist = PVE::Cluster::get_nodelist();
2295     my $nodehash = { map { $_ => 1 } @$nodelist };
2296     my $nodename = PVE::INotify::nodename();
2297
2298     foreach_drive($conf, sub {
2299         my ($ds, $drive) = @_;
2300
2301         my $volid = $drive->{file};
2302         return if !$volid;
2303
2304         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2305         if ($storeid) {
2306             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2307             if ($scfg->{disable}) {
2308                 $nodehash = {};
2309             } elsif (my $avail = $scfg->{nodes}) {
2310                 foreach my $node (keys %$nodehash) {
2311                     delete $nodehash->{$node} if !$avail->{$node};
2312                 }
2313             } elsif (!$scfg->{shared}) {
2314                 foreach my $node (keys %$nodehash) {
2315                     delete $nodehash->{$node} if $node ne $nodename
2316                 }
2317             }
2318         }
2319     });
2320
2321     return $nodehash
2322 }
2323
2324 sub check_cmdline {
2325     my ($pidfile, $pid) = @_;
2326
2327     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2328     if (defined($fh)) {
2329         my $line = <$fh>;
2330         $fh->close;
2331         return undef if !$line;
2332         my @param = split(/\0/, $line);
2333
2334         my $cmd = $param[0];
2335         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2336
2337         for (my $i = 0; $i < scalar (@param); $i++) {
2338             my $p = $param[$i];
2339             next if !$p;
2340             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2341                 my $p = $param[$i+1];
2342                 return 1 if $p && ($p eq $pidfile);
2343                 return undef;
2344             }
2345         }
2346     }
2347     return undef;
2348 }
2349
2350 sub check_running {
2351     my ($vmid, $nocheck, $node) = @_;
2352
2353     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2354
2355     die "unable to find configuration file for VM $vmid - no such machine\n"
2356         if !$nocheck && ! -f $filename;
2357
2358     my $pidfile = pidfile_name($vmid);
2359
2360     if (my $fd = IO::File->new("<$pidfile")) {
2361         my $st = stat($fd);
2362         my $line = <$fd>;
2363         close($fd);
2364
2365         my $mtime = $st->mtime;
2366         if ($mtime > time()) {
2367             warn "file '$filename' modified in future\n";
2368         }
2369
2370         if ($line =~ m/^(\d+)$/) {
2371             my $pid = $1;
2372             if (check_cmdline($pidfile, $pid)) {
2373                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2374                     return $pid;
2375                 }
2376             }
2377         }
2378     }
2379
2380     return undef;
2381 }
2382
2383 sub vzlist {
2384
2385     my $vzlist = config_list();
2386
2387     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2388
2389     while (defined(my $de = $fd->read)) {
2390         next if $de !~ m/^(\d+)\.pid$/;
2391         my $vmid = $1;
2392         next if !defined($vzlist->{$vmid});
2393         if (my $pid = check_running($vmid)) {
2394             $vzlist->{$vmid}->{pid} = $pid;
2395         }
2396     }
2397
2398     return $vzlist;
2399 }
2400
2401 sub disksize {
2402     my ($storecfg, $conf) = @_;
2403
2404     my $bootdisk = $conf->{bootdisk};
2405     return undef if !$bootdisk;
2406     return undef if !is_valid_drivename($bootdisk);
2407
2408     return undef if !$conf->{$bootdisk};
2409
2410     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2411     return undef if !defined($drive);
2412
2413     return undef if drive_is_cdrom($drive);
2414
2415     my $volid = $drive->{file};
2416     return undef if !$volid;
2417
2418     return $drive->{size};
2419 }
2420
2421 my $last_proc_pid_stat;
2422
2423 # get VM status information
2424 # This must be fast and should not block ($full == false)
2425 # We only query KVM using QMP if $full == true (this can be slow)
2426 sub vmstatus {
2427     my ($opt_vmid, $full) = @_;
2428
2429     my $res = {};
2430
2431     my $storecfg = PVE::Storage::config();
2432
2433     my $list = vzlist();
2434     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2435
2436     my $cpucount = $cpuinfo->{cpus} || 1;
2437
2438     foreach my $vmid (keys %$list) {
2439         next if $opt_vmid && ($vmid ne $opt_vmid);
2440
2441         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2442         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2443
2444         my $d = {};
2445         $d->{pid} = $list->{$vmid}->{pid};
2446
2447         # fixme: better status?
2448         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2449
2450         my $size = disksize($storecfg, $conf);
2451         if (defined($size)) {
2452             $d->{disk} = 0; # no info available
2453             $d->{maxdisk} = $size;
2454         } else {
2455             $d->{disk} = 0;
2456             $d->{maxdisk} = 0;
2457         }
2458
2459         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2460         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2461         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2462
2463         $d->{name} = $conf->{name} || "VM $vmid";
2464         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2465
2466         if ($conf->{balloon}) {
2467             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2468             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2469         }
2470
2471         $d->{uptime} = 0;
2472         $d->{cpu} = 0;
2473         $d->{mem} = 0;
2474
2475         $d->{netout} = 0;
2476         $d->{netin} = 0;
2477
2478         $d->{diskread} = 0;
2479         $d->{diskwrite} = 0;
2480
2481         $d->{template} = PVE::QemuConfig->is_template($conf);
2482
2483         $res->{$vmid} = $d;
2484     }
2485
2486     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2487     foreach my $dev (keys %$netdev) {
2488         next if $dev !~ m/^tap([1-9]\d*)i/;
2489         my $vmid = $1;
2490         my $d = $res->{$vmid};
2491         next if !$d;
2492
2493         $d->{netout} += $netdev->{$dev}->{receive};
2494         $d->{netin} += $netdev->{$dev}->{transmit};
2495
2496         if ($full) {
2497             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2498             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2499         }
2500
2501     }
2502
2503     my $ctime = gettimeofday;
2504
2505     foreach my $vmid (keys %$list) {
2506
2507         my $d = $res->{$vmid};
2508         my $pid = $d->{pid};
2509         next if !$pid;
2510
2511         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2512         next if !$pstat; # not running
2513
2514         my $used = $pstat->{utime} + $pstat->{stime};
2515
2516         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2517
2518         if ($pstat->{vsize}) {
2519             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2520         }
2521
2522         my $old = $last_proc_pid_stat->{$pid};
2523         if (!$old) {
2524             $last_proc_pid_stat->{$pid} = {
2525                 time => $ctime,
2526                 used => $used,
2527                 cpu => 0,
2528             };
2529             next;
2530         }
2531
2532         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2533
2534         if ($dtime > 1000) {
2535             my $dutime = $used -  $old->{used};
2536
2537             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2538             $last_proc_pid_stat->{$pid} = {
2539                 time => $ctime,
2540                 used => $used,
2541                 cpu => $d->{cpu},
2542             };
2543         } else {
2544             $d->{cpu} = $old->{cpu};
2545         }
2546     }
2547
2548     return $res if !$full;
2549
2550     my $qmpclient = PVE::QMPClient->new();
2551
2552     my $ballooncb = sub {
2553         my ($vmid, $resp) = @_;
2554
2555         my $info = $resp->{'return'};
2556         return if !$info->{max_mem};
2557
2558         my $d = $res->{$vmid};
2559
2560         # use memory assigned to VM
2561         $d->{maxmem} = $info->{max_mem};
2562         $d->{balloon} = $info->{actual};
2563
2564         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2565             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2566             $d->{freemem} = $info->{free_mem};
2567         }
2568
2569         $d->{ballooninfo} = $info;
2570     };
2571
2572     my $blockstatscb = sub {
2573         my ($vmid, $resp) = @_;
2574         my $data = $resp->{'return'} || [];
2575         my $totalrdbytes = 0;
2576         my $totalwrbytes = 0;
2577
2578         for my $blockstat (@$data) {
2579             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2580             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2581
2582             $blockstat->{device} =~ s/drive-//;
2583             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2584         }
2585         $res->{$vmid}->{diskread} = $totalrdbytes;
2586         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2587     };
2588
2589     my $statuscb = sub {
2590         my ($vmid, $resp) = @_;
2591
2592         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2593         # this fails if ballon driver is not loaded, so this must be
2594         # the last commnand (following command are aborted if this fails).
2595         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2596
2597         my $status = 'unknown';
2598         if (!defined($status = $resp->{'return'}->{status})) {
2599             warn "unable to get VM status\n";
2600             return;
2601         }
2602
2603         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2604     };
2605
2606     foreach my $vmid (keys %$list) {
2607         next if $opt_vmid && ($vmid ne $opt_vmid);
2608         next if !$res->{$vmid}->{pid}; # not running
2609         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2610     }
2611
2612     $qmpclient->queue_execute(undef, 1);
2613
2614     foreach my $vmid (keys %$list) {
2615         next if $opt_vmid && ($vmid ne $opt_vmid);
2616         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2617     }
2618
2619     return $res;
2620 }
2621
2622 sub foreach_dimm {
2623     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2624
2625     my $dimm_id = 0;
2626     my $current_size = 1024;
2627     my $dimm_size = 512;
2628     return if $current_size == $memory;
2629
2630     for (my $j = 0; $j < 8; $j++) {
2631         for (my $i = 0; $i < 32; $i++) {
2632             my $name = "dimm${dimm_id}";
2633             $dimm_id++;
2634             my $numanode = $i % $sockets;
2635             $current_size += $dimm_size;
2636             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2637             return  $current_size if $current_size >= $memory;
2638         }
2639         $dimm_size *= 2;
2640     }
2641 }
2642
2643 sub foreach_reverse_dimm {
2644     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2645
2646     my $dimm_id = 253;
2647     my $current_size = 4177920;
2648     my $dimm_size = 65536;
2649     return if $current_size == $memory;
2650
2651     for (my $j = 0; $j < 8; $j++) {
2652         for (my $i = 0; $i < 32; $i++) {
2653             my $name = "dimm${dimm_id}";
2654             $dimm_id--;
2655             my $numanode = $i % $sockets;
2656             $current_size -= $dimm_size;
2657             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2658             return  $current_size if $current_size <= $memory;
2659         }
2660         $dimm_size /= 2;
2661     }
2662 }
2663
2664 sub foreach_drive {
2665     my ($conf, $func) = @_;
2666
2667     foreach my $ds (valid_drive_names()) {
2668         next if !defined($conf->{$ds});
2669
2670         my $drive = parse_drive($ds, $conf->{$ds});
2671         next if !$drive;
2672
2673         &$func($ds, $drive);
2674     }
2675 }
2676
2677 sub foreach_volid {
2678     my ($conf, $func) = @_;
2679
2680     my $volhash = {};
2681
2682     my $test_volid = sub {
2683         my ($volid, $is_cdrom) = @_;
2684
2685         return if !$volid;
2686
2687         $volhash->{$volid} = $is_cdrom || 0;
2688     };
2689
2690     foreach_drive($conf, sub {
2691         my ($ds, $drive) = @_;
2692         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2693     });
2694
2695     foreach my $snapname (keys %{$conf->{snapshots}}) {
2696         my $snap = $conf->{snapshots}->{$snapname};
2697         &$test_volid($snap->{vmstate}, 0);
2698         foreach_drive($snap, sub {
2699             my ($ds, $drive) = @_;
2700             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2701         });
2702     }
2703
2704     foreach my $volid (keys %$volhash) {
2705         &$func($volid, $volhash->{$volid});
2706     }
2707 }
2708
2709 sub vga_conf_has_spice {
2710     my ($vga) = @_;
2711
2712     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2713
2714     return $1 || 1;
2715 }
2716
2717 sub config_to_command {
2718     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2719
2720     my $cmd = [];
2721     my $globalFlags = [];
2722     my $machineFlags = [];
2723     my $rtcFlags = [];
2724     my $cpuFlags = [];
2725     my $devices = [];
2726     my $pciaddr = '';
2727     my $bridges = {};
2728     my $kvmver = kvm_user_version();
2729     my $vernum = 0; # unknown
2730     my $ostype = $conf->{ostype};
2731     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2732         $vernum = $1*1000000+$2*1000;
2733     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2734         $vernum = $1*1000000+$2*1000+$3;
2735     }
2736
2737     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2738
2739     my $have_ovz = -f '/proc/vz/vestat';
2740
2741     my $q35 = machine_type_is_q35($conf);
2742     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2743     my $machine_type = $forcemachine || $conf->{machine};
2744     my $use_old_bios_files = undef;
2745     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2746
2747     my $cpuunits = defined($conf->{cpuunits}) ?
2748             $conf->{cpuunits} : $defaults->{cpuunits};
2749
2750     push @$cmd, '/usr/bin/systemd-run';
2751     push @$cmd, '--scope';
2752     push @$cmd, '--slice', "qemu";
2753     push @$cmd, '--unit', $vmid;
2754     # set KillMode=none, so that systemd don't kill those scopes
2755     # at shutdown (pve-manager service should stop the VMs instead)
2756     push @$cmd, '-p', "KillMode=none";
2757     push @$cmd, '-p', "CPUShares=$cpuunits";
2758     if ($conf->{cpulimit}) {
2759         my $cpulimit = int($conf->{cpulimit} * 100);
2760         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2761     }
2762
2763     push @$cmd, '/usr/bin/kvm';
2764
2765     push @$cmd, '-id', $vmid;
2766
2767     my $use_virtio = 0;
2768
2769     my $qmpsocket = qmp_socket($vmid);
2770     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2771     push @$cmd, '-mon', "chardev=qmp,mode=control";
2772
2773
2774     push @$cmd, '-pidfile' , pidfile_name($vmid);
2775
2776     push @$cmd, '-daemonize';
2777
2778     if ($conf->{smbios1}) {
2779         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2780     }
2781
2782     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2783         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2784         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2785         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2786         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2787         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2788         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2789     }
2790
2791     if ($q35) {
2792         # the q35 chipset support native usb2, so we enable usb controller
2793         # by default for this machine type
2794         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2795     } else {
2796         $pciaddr = print_pci_addr("piix3", $bridges);
2797         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2798
2799         my $use_usb2 = 0;
2800         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2801             next if !$conf->{"usb$i"};
2802             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2803             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2804             $use_usb2 = 1;
2805         }
2806         # include usb device config
2807         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2808     }
2809
2810     # add usb3 controller if needed
2811
2812     my $use_usb3 = 0;
2813     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2814         next if !$conf->{"usb$i"};
2815         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2816         next if !$d || !$d->{usb3};
2817         $use_usb3 = 1;
2818     }
2819
2820     $pciaddr = print_pci_addr("xhci", $bridges);
2821     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2822
2823     my $vga = $conf->{vga};
2824
2825     my $qxlnum = vga_conf_has_spice($vga);
2826     $vga = 'qxl' if $qxlnum;
2827
2828     if (!$vga) {
2829         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2830                                 $conf->{ostype} eq 'win7' ||
2831                                 $conf->{ostype} eq 'w2k8')) {
2832             $vga = 'std';
2833         } else {
2834             $vga = 'cirrus';
2835         }
2836     }
2837
2838     # enable absolute mouse coordinates (needed by vnc)
2839     my $tablet;
2840     if (defined($conf->{tablet})) {
2841         $tablet = $conf->{tablet};
2842     } else {
2843         $tablet = $defaults->{tablet};
2844         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2845         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2846     }
2847
2848     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2849
2850     my $kvm_off = 0;
2851     # host pci devices
2852     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2853         my $d = parse_hostpci($conf->{"hostpci$i"});
2854         next if !$d;
2855
2856         my $pcie = $d->{pcie};
2857         if($pcie){
2858             die "q35 machine model is not enabled" if !$q35;
2859             $pciaddr = print_pcie_addr("hostpci$i");
2860         }else{
2861             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2862         }
2863
2864         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2865         my $xvga = '';
2866         if ($d->{'x-vga'}) {
2867             $xvga = ',x-vga=on';
2868             $kvm_off = 1;
2869             $vga = 'none';
2870             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2871                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2872             }
2873             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2874                 $xvga = "";
2875             }
2876         }
2877         my $pcidevices = $d->{pciid};
2878         my $multifunction = 1 if @$pcidevices > 1;
2879
2880         my $j=0;
2881         foreach my $pcidevice (@$pcidevices) {
2882
2883             my $id = "hostpci$i";
2884             $id .= ".$j" if $multifunction;
2885             my $addr = $pciaddr;
2886             $addr .= ".$j" if $multifunction;
2887             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2888
2889             if($j == 0){
2890                 $devicestr .= "$rombar$xvga";
2891                 $devicestr .= ",multifunction=on" if $multifunction;
2892             }
2893
2894             push @$devices, '-device', $devicestr;
2895             $j++;
2896         }
2897     }
2898
2899     # usb devices
2900     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2901         next if !$conf->{"usb$i"};
2902         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2903         next if !$d;
2904
2905         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2906         my $usbbus = '';
2907         if (defined($d->{usb3}) && $d->{usb3}) {
2908             $usbbus = ',bus=xhci.0';
2909         }
2910
2911         if (defined($d->{host})) {
2912             $d = parse_usb_device($d->{host});
2913             if (defined($d->{vendorid}) && defined($d->{productid})) {
2914                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2915             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2916                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2917             } elsif (defined($d->{spice}) && $d->{spice}) {
2918                 # usb redir support for spice, currently no usb3
2919                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2920                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2921             }
2922         }
2923     }
2924
2925     # serial devices
2926     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2927         if (my $path = $conf->{"serial$i"}) {
2928             if ($path eq 'socket') {
2929                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2930                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2931                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2932             } else {
2933                 die "no such serial device\n" if ! -c $path;
2934                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2935                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2936             }
2937         }
2938     }
2939
2940     # parallel devices
2941     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2942         if (my $path = $conf->{"parallel$i"}) {
2943             die "no such parallel device\n" if ! -c $path;
2944             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2945             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2946             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2947         }
2948     }
2949
2950     my $vmname = $conf->{name} || "vm$vmid";
2951
2952     push @$cmd, '-name', $vmname;
2953
2954     my $sockets = 1;
2955     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2956     $sockets = $conf->{sockets} if  $conf->{sockets};
2957
2958     my $cores = $conf->{cores} || 1;
2959
2960     my $maxcpus = $sockets * $cores;
2961
2962     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2963
2964     my $allowed_vcpus = $cpuinfo->{cpus};
2965
2966     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2967         if ($allowed_vcpus < $maxcpus);
2968
2969     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2970
2971     push @$cmd, '-nodefaults';
2972
2973     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2974
2975     my $bootindex_hash = {};
2976     my $i = 1;
2977     foreach my $o (split(//, $bootorder)) {
2978         $bootindex_hash->{$o} = $i*100;
2979         $i++;
2980     }
2981
2982     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2983
2984     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2985
2986     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2987
2988     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2989
2990     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2991         my $socket = vnc_socket($vmid);
2992         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2993     } else {
2994         push @$cmd, '-nographic';
2995     }
2996
2997     # time drift fix
2998     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2999
3000     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3001     my $useLocaltime = $conf->{localtime};
3002
3003     if ($ostype) {
3004         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3005
3006         if ($ostype =~ m/^w/) { # windows
3007             $useLocaltime = 1 if !defined($conf->{localtime});
3008
3009             # use time drift fix when acpi is enabled
3010             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3011                 $tdf = 1 if !defined($conf->{tdf});
3012             }
3013         }
3014
3015         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3016             $ostype eq 'wvista') {
3017             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3018             push @$cmd, '-no-hpet';
3019             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3020                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3021                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3022                 push @$cpuFlags , 'hv_time' if !$nokvm;
3023
3024             } else {
3025                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3026             }
3027         }
3028
3029         if ($ostype eq 'win7' || $ostype eq 'win8') {
3030             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3031         }
3032     }
3033
3034     push @$rtcFlags, 'driftfix=slew' if $tdf;
3035
3036     if ($nokvm) {
3037         push @$machineFlags, 'accel=tcg';
3038     } else {
3039         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3040     }
3041
3042     if ($machine_type) {
3043         push @$machineFlags, "type=${machine_type}";
3044     }
3045
3046     if ($conf->{startdate}) {
3047         push @$rtcFlags, "base=$conf->{startdate}";
3048     } elsif ($useLocaltime) {
3049         push @$rtcFlags, 'base=localtime';
3050     }
3051
3052     my $cpu = $nokvm ? "qemu64" : "kvm64";
3053     if (my $cputype = $conf->{cpu}) {
3054         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3055             or die "Cannot parse cpu description: $cputype\n";
3056         $cpu = $cpuconf->{cputype};
3057         $kvm_off = 1 if $cpuconf->{hidden};
3058     }
3059
3060     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3061
3062     push @$cpuFlags , '-x2apic'
3063         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3064
3065     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3066
3067     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3068
3069     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3070
3071         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3072         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3073     }
3074
3075     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3076
3077     push @$cpuFlags, 'kvm=off' if $kvm_off;
3078
3079     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3080
3081     push @$cmd, '-cpu', $cpu;
3082
3083     my $memory = $conf->{memory} || $defaults->{memory};
3084     my $static_memory = 0;
3085     my $dimm_memory = 0;
3086
3087     if ($hotplug_features->{memory}) {
3088         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3089         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3090         $static_memory = $STATICMEM;
3091         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3092         $dimm_memory = $memory - $static_memory;
3093         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3094
3095     } else {
3096
3097         $static_memory = $memory;
3098         push @$cmd, '-m', $static_memory;
3099     }
3100
3101     if ($conf->{numa}) {
3102
3103         my $numa_totalmemory = undef;
3104         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3105             next if !$conf->{"numa$i"};
3106             my $numa = parse_numa($conf->{"numa$i"});
3107             next if !$numa;
3108             # memory
3109             die "missing numa node$i memory value\n" if !$numa->{memory};
3110             my $numa_memory = $numa->{memory};
3111             $numa_totalmemory += $numa_memory;
3112             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3113
3114             # cpus
3115             my $cpulists = $numa->{cpus};
3116             die "missing numa node$i cpus\n" if !defined($cpulists);
3117             my $cpus = join(',', map {
3118                 my ($start, $end) = @$_;
3119                 defined($end) ? "$start-$end" : $start
3120             } @$cpulists);
3121
3122             # hostnodes
3123             my $hostnodelists = $numa->{hostnodes};
3124             if (defined($hostnodelists)) {
3125                 my $hostnodes;
3126                 foreach my $hostnoderange (@$hostnodelists) {
3127                     my ($start, $end) = @$hostnoderange;
3128                     $hostnodes .= ',' if $hostnodes;
3129                     $hostnodes .= $start;
3130                     $hostnodes .= "-$end" if defined($end);
3131                     $end //= $start;
3132                     for (my $i = $start; $i <= $end; ++$i ) {
3133                         die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3134                     }
3135                 }
3136
3137                 # policy
3138                 my $policy = $numa->{policy};
3139                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3140                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3141             }
3142
3143             push @$cmd, '-object', $numa_object;
3144             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3145         }
3146
3147         die "total memory for NUMA nodes must be equal to vm static memory\n"
3148             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3149
3150         #if no custom tology, we split memory and cores across numa nodes
3151         if(!$numa_totalmemory) {
3152
3153             my $numa_memory = ($static_memory / $sockets) . "M";
3154
3155             for (my $i = 0; $i < $sockets; $i++)  {
3156
3157                 my $cpustart = ($cores * $i);
3158                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3159                 my $cpus = $cpustart;
3160                 $cpus .= "-$cpuend" if $cpuend;
3161
3162                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3163                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3164             }
3165         }
3166     }
3167
3168     if ($hotplug_features->{memory}) {
3169         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3170             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3171             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3172             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3173
3174             #if dimm_memory is not aligned to dimm map
3175             if($current_size > $memory) {
3176                  $conf->{memory} = $current_size;
3177                  PVE::QemuConfig->write_config($vmid, $conf);
3178             }
3179         });
3180     }
3181
3182     push @$cmd, '-S' if $conf->{freeze};
3183
3184     # set keyboard layout
3185     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3186     push @$cmd, '-k', $kb if $kb;
3187
3188     # enable sound
3189     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3190     #push @$cmd, '-soundhw', 'es1370';
3191     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3192
3193     if($conf->{agent}) {
3194         my $qgasocket = qmp_socket($vmid, 1);
3195         my $pciaddr = print_pci_addr("qga0", $bridges);
3196         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3197         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3198         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3199     }
3200
3201     my $spice_port;
3202
3203     if ($qxlnum) {
3204         if ($qxlnum > 1) {
3205             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3206                 for(my $i = 1; $i < $qxlnum; $i++){
3207                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3208                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3209                 }
3210             } else {
3211                 # assume other OS works like Linux
3212                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3213                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3214             }
3215         }
3216
3217         my $pciaddr = print_pci_addr("spice", $bridges);
3218
3219         my $nodename = PVE::INotify::nodename();
3220         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3221         $spice_port = PVE::Tools::next_spice_port($pfamily);
3222
3223         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3224
3225         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3226         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3227         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3228     }
3229
3230     # enable balloon by default, unless explicitly disabled
3231     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3232         $pciaddr = print_pci_addr("balloon0", $bridges);
3233         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3234     }
3235
3236     if ($conf->{watchdog}) {
3237         my $wdopts = parse_watchdog($conf->{watchdog});
3238         $pciaddr = print_pci_addr("watchdog", $bridges);
3239         my $watchdog = $wdopts->{model} || 'i6300esb';
3240         push @$devices, '-device', "$watchdog$pciaddr";
3241         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3242     }
3243
3244     my $vollist = [];
3245     my $scsicontroller = {};
3246     my $ahcicontroller = {};
3247     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3248
3249     # Add iscsi initiator name if available
3250     if (my $initiator = get_initiator_name()) {
3251         push @$devices, '-iscsi', "initiator-name=$initiator";
3252     }
3253
3254     foreach_drive($conf, sub {
3255         my ($ds, $drive) = @_;
3256
3257         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3258             push @$vollist, $drive->{file};
3259         }
3260
3261         $use_virtio = 1 if $ds =~ m/^virtio/;
3262
3263         if (drive_is_cdrom ($drive)) {
3264             if ($bootindex_hash->{d}) {
3265                 $drive->{bootindex} = $bootindex_hash->{d};
3266                 $bootindex_hash->{d} += 1;
3267             }
3268         } else {
3269             if ($bootindex_hash->{c}) {
3270                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3271                 $bootindex_hash->{c} += 1;
3272             }
3273         }
3274
3275         if($drive->{interface} eq 'virtio'){
3276            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3277         }
3278
3279         if ($drive->{interface} eq 'scsi') {
3280
3281             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3282
3283             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3284             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3285
3286             my $iothread = '';
3287             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3288                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3289                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3290             }
3291
3292             my $queues = '';
3293             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3294                 $queues = ",num_queues=$drive->{queues}";
3295             }
3296
3297             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3298             $scsicontroller->{$controller}=1;
3299         }
3300
3301         if ($drive->{interface} eq 'sata') {
3302            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3303            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3304            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3305            $ahcicontroller->{$controller}=1;
3306         }
3307
3308         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3309         push @$devices, '-drive',$drive_cmd;
3310         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3311     });
3312
3313     for (my $i = 0; $i < $MAX_NETS; $i++) {
3314          next if !$conf->{"net$i"};
3315          my $d = parse_net($conf->{"net$i"});
3316          next if !$d;
3317
3318          $use_virtio = 1 if $d->{model} eq 'virtio';
3319
3320          if ($bootindex_hash->{n}) {
3321             $d->{bootindex} = $bootindex_hash->{n};
3322             $bootindex_hash->{n} += 1;
3323          }
3324
3325          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3326          push @$devices, '-netdev', $netdevfull;
3327
3328          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3329          push @$devices, '-device', $netdevicefull;
3330     }
3331
3332     if (!$q35) {
3333         # add pci bridges
3334         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3335            $bridges->{1} = 1;
3336            $bridges->{2} = 1;
3337         }
3338
3339         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3340
3341         while (my ($k, $v) = each %$bridges) {
3342             $pciaddr = print_pci_addr("pci.$k");
3343             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3344         }
3345     }
3346
3347     # add custom args
3348     if ($conf->{args}) {
3349         my $aa = PVE::Tools::split_args($conf->{args});
3350         push @$cmd, @$aa;
3351     }
3352
3353     push @$cmd, @$devices;
3354     push @$cmd, '-rtc', join(',', @$rtcFlags)
3355         if scalar(@$rtcFlags);
3356     push @$cmd, '-machine', join(',', @$machineFlags)
3357         if scalar(@$machineFlags);
3358     push @$cmd, '-global', join(',', @$globalFlags)
3359         if scalar(@$globalFlags);
3360
3361     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3362 }
3363
3364 sub vnc_socket {
3365     my ($vmid) = @_;
3366     return "${var_run_tmpdir}/$vmid.vnc";
3367 }
3368
3369 sub spice_port {
3370     my ($vmid) = @_;
3371
3372     my $res = vm_mon_cmd($vmid, 'query-spice');
3373
3374     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3375 }
3376
3377 sub qmp_socket {
3378     my ($vmid, $qga) = @_;
3379     my $sockettype = $qga ? 'qga' : 'qmp';
3380     return "${var_run_tmpdir}/$vmid.$sockettype";
3381 }
3382
3383 sub pidfile_name {
3384     my ($vmid) = @_;
3385     return "${var_run_tmpdir}/$vmid.pid";
3386 }
3387
3388 sub vm_devices_list {
3389     my ($vmid) = @_;
3390
3391     my $res = vm_mon_cmd($vmid, 'query-pci');
3392     my $devices = {};
3393     foreach my $pcibus (@$res) {
3394         foreach my $device (@{$pcibus->{devices}}) {
3395             next if !$device->{'qdev_id'};
3396             if ($device->{'pci_bridge'}) {
3397                 $devices->{$device->{'qdev_id'}} = 1;
3398                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3399                     next if !$bridge_device->{'qdev_id'};
3400                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3401                     $devices->{$device->{'qdev_id'}}++;
3402                 }
3403             } else {
3404                 $devices->{$device->{'qdev_id'}} = 1;
3405             }
3406         }
3407     }
3408
3409     my $resblock = vm_mon_cmd($vmid, 'query-block');
3410     foreach my $block (@$resblock) {
3411         if($block->{device} =~ m/^drive-(\S+)/){
3412                 $devices->{$1} = 1;
3413         }
3414     }
3415
3416     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3417     foreach my $mice (@$resmice) {
3418         if ($mice->{name} eq 'QEMU HID Tablet') {
3419             $devices->{tablet} = 1;
3420             last;
3421         }
3422     }
3423
3424     return $devices;
3425 }
3426
3427 sub vm_deviceplug {
3428     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3429
3430     my $q35 = machine_type_is_q35($conf);
3431
3432     my $devices_list = vm_devices_list($vmid);
3433     return 1 if defined($devices_list->{$deviceid});
3434
3435     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3436
3437     if ($deviceid eq 'tablet') {
3438
3439         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3440
3441     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3442
3443         qemu_iothread_add($vmid, $deviceid, $device);
3444
3445         qemu_driveadd($storecfg, $vmid, $device);
3446         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3447
3448         qemu_deviceadd($vmid, $devicefull);
3449         eval { qemu_deviceaddverify($vmid, $deviceid); };
3450         if (my $err = $@) {
3451             eval { qemu_drivedel($vmid, $deviceid); };
3452             warn $@ if $@;
3453             die $err;
3454         }
3455
3456     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3457
3458
3459         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3460         my $pciaddr = print_pci_addr($deviceid);
3461         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3462
3463         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3464
3465         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3466             qemu_iothread_add($vmid, $deviceid, $device);
3467             $devicefull .= ",iothread=iothread-$deviceid";
3468         }
3469
3470         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3471             $devicefull .= ",num_queues=$device->{queues}";
3472         }
3473
3474         qemu_deviceadd($vmid, $devicefull);
3475         qemu_deviceaddverify($vmid, $deviceid);
3476
3477     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3478
3479         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3480         qemu_driveadd($storecfg, $vmid, $device);
3481
3482         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3483         eval { qemu_deviceadd($vmid, $devicefull); };
3484         if (my $err = $@) {
3485             eval { qemu_drivedel($vmid, $deviceid); };
3486             warn $@ if $@;
3487             die $err;
3488         }
3489
3490     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3491
3492         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3493
3494         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3495         my $use_old_bios_files = undef;
3496         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3497
3498         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3499         qemu_deviceadd($vmid, $netdevicefull);
3500         eval { qemu_deviceaddverify($vmid, $deviceid); };
3501         if (my $err = $@) {
3502             eval { qemu_netdevdel($vmid, $deviceid); };
3503             warn $@ if $@;
3504             die $err;
3505         }
3506
3507     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3508
3509         my $bridgeid = $2;
3510         my $pciaddr = print_pci_addr($deviceid);
3511         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3512
3513         qemu_deviceadd($vmid, $devicefull);
3514         qemu_deviceaddverify($vmid, $deviceid);
3515
3516     } else {
3517         die "can't hotplug device '$deviceid'\n";
3518     }
3519
3520     return 1;
3521 }
3522
3523 # fixme: this should raise exceptions on error!
3524 sub vm_deviceunplug {
3525     my ($vmid, $conf, $deviceid) = @_;
3526
3527     my $devices_list = vm_devices_list($vmid);
3528     return 1 if !defined($devices_list->{$deviceid});
3529
3530     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3531
3532     if ($deviceid eq 'tablet') {
3533
3534         qemu_devicedel($vmid, $deviceid);
3535
3536     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3537
3538         qemu_devicedel($vmid, $deviceid);
3539         qemu_devicedelverify($vmid, $deviceid);
3540         qemu_drivedel($vmid, $deviceid);
3541         qemu_iothread_del($conf, $vmid, $deviceid);
3542
3543     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3544
3545         qemu_devicedel($vmid, $deviceid);
3546         qemu_devicedelverify($vmid, $deviceid);
3547         qemu_iothread_del($conf, $vmid, $deviceid);
3548
3549     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3550
3551         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3552         my $device = parse_drive($deviceid, $conf->{$deviceid});
3553         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3554
3555         qemu_devicedel($vmid, $deviceid);
3556         qemu_drivedel($vmid, $deviceid);
3557         qemu_deletescsihw($conf, $vmid, $deviceid);
3558
3559     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3560
3561         qemu_devicedel($vmid, $deviceid);
3562         qemu_devicedelverify($vmid, $deviceid);
3563         qemu_netdevdel($vmid, $deviceid);
3564
3565     } else {
3566         die "can't unplug device '$deviceid'\n";
3567     }
3568
3569     return 1;
3570 }
3571
3572 sub qemu_deviceadd {
3573     my ($vmid, $devicefull) = @_;
3574
3575     $devicefull = "driver=".$devicefull;
3576     my %options =  split(/[=,]/, $devicefull);
3577
3578     vm_mon_cmd($vmid, "device_add" , %options);
3579 }
3580
3581 sub qemu_devicedel {
3582     my ($vmid, $deviceid) = @_;
3583
3584     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3585 }
3586
3587 sub qemu_iothread_add {
3588     my($vmid, $deviceid, $device) = @_;
3589
3590     if ($device->{iothread}) {
3591         my $iothreads = vm_iothreads_list($vmid);
3592         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3593     }
3594 }
3595
3596 sub qemu_iothread_del {
3597     my($conf, $vmid, $deviceid) = @_;
3598
3599     my $device = parse_drive($deviceid, $conf->{$deviceid});
3600     if ($device->{iothread}) {
3601         my $iothreads = vm_iothreads_list($vmid);
3602         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3603     }
3604 }
3605
3606 sub qemu_objectadd {
3607     my($vmid, $objectid, $qomtype) = @_;
3608
3609     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3610
3611     return 1;
3612 }
3613
3614 sub qemu_objectdel {
3615     my($vmid, $objectid) = @_;
3616
3617     vm_mon_cmd($vmid, "object-del", id => $objectid);
3618
3619     return 1;
3620 }
3621
3622 sub qemu_driveadd {
3623     my ($storecfg, $vmid, $device) = @_;
3624
3625     my $drive = print_drive_full($storecfg, $vmid, $device);
3626     $drive =~ s/\\/\\\\/g;
3627     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3628
3629     # If the command succeeds qemu prints: "OK"
3630     return 1 if $ret =~ m/OK/s;
3631
3632     die "adding drive failed: $ret\n";
3633 }
3634
3635 sub qemu_drivedel {
3636     my($vmid, $deviceid) = @_;
3637
3638     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3639     $ret =~ s/^\s+//;
3640
3641     return 1 if $ret eq "";
3642
3643     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3644     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3645
3646     die "deleting drive $deviceid failed : $ret\n";
3647 }
3648
3649 sub qemu_deviceaddverify {
3650     my ($vmid, $deviceid) = @_;
3651
3652     for (my $i = 0; $i <= 5; $i++) {
3653          my $devices_list = vm_devices_list($vmid);
3654          return 1 if defined($devices_list->{$deviceid});
3655          sleep 1;
3656     }
3657
3658     die "error on hotplug device '$deviceid'\n";
3659 }
3660
3661
3662 sub qemu_devicedelverify {
3663     my ($vmid, $deviceid) = @_;
3664
3665     # need to verify that the device is correctly removed as device_del
3666     # is async and empty return is not reliable
3667
3668     for (my $i = 0; $i <= 5; $i++) {
3669          my $devices_list = vm_devices_list($vmid);
3670          return 1 if !defined($devices_list->{$deviceid});
3671          sleep 1;
3672     }
3673
3674     die "error on hot-unplugging device '$deviceid'\n";
3675 }
3676
3677 sub qemu_findorcreatescsihw {
3678     my ($storecfg, $conf, $vmid, $device) = @_;
3679
3680     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3681
3682     my $scsihwid="$controller_prefix$controller";
3683     my $devices_list = vm_devices_list($vmid);
3684
3685     if(!defined($devices_list->{$scsihwid})) {
3686         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3687     }
3688
3689     return 1;
3690 }
3691
3692 sub qemu_deletescsihw {
3693     my ($conf, $vmid, $opt) = @_;
3694
3695     my $device = parse_drive($opt, $conf->{$opt});
3696
3697     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3698         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3699         return 1;
3700     }
3701
3702     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3703
3704     my $devices_list = vm_devices_list($vmid);
3705     foreach my $opt (keys %{$devices_list}) {
3706         if (PVE::QemuServer::is_valid_drivename($opt)) {
3707             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3708             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3709                 return 1;
3710             }
3711         }
3712     }
3713
3714     my $scsihwid="scsihw$controller";
3715
3716     vm_deviceunplug($vmid, $conf, $scsihwid);
3717
3718     return 1;
3719 }
3720
3721 sub qemu_add_pci_bridge {
3722     my ($storecfg, $conf, $vmid, $device) = @_;
3723
3724     my $bridges = {};
3725
3726     my $bridgeid;
3727
3728     print_pci_addr($device, $bridges);
3729
3730     while (my ($k, $v) = each %$bridges) {
3731         $bridgeid = $k;
3732     }
3733     return 1 if !defined($bridgeid) || $bridgeid < 1;
3734
3735     my $bridge = "pci.$bridgeid";
3736     my $devices_list = vm_devices_list($vmid);
3737
3738     if (!defined($devices_list->{$bridge})) {
3739         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3740     }
3741
3742     return 1;
3743 }
3744
3745 sub qemu_set_link_status {
3746     my ($vmid, $device, $up) = @_;
3747
3748     vm_mon_cmd($vmid, "set_link", name => $device,
3749                up => $up ? JSON::true : JSON::false);
3750 }
3751
3752 sub qemu_netdevadd {
3753     my ($vmid, $conf, $device, $deviceid) = @_;
3754
3755     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3756     my %options =  split(/[=,]/, $netdev);
3757
3758     vm_mon_cmd($vmid, "netdev_add",  %options);
3759     return 1;
3760 }
3761
3762 sub qemu_netdevdel {
3763     my ($vmid, $deviceid) = @_;
3764
3765     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3766 }
3767
3768 sub qemu_cpu_hotplug {
3769     my ($vmid, $conf, $vcpus) = @_;
3770
3771     my $sockets = 1;
3772     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3773     $sockets = $conf->{sockets} if  $conf->{sockets};
3774     my $cores = $conf->{cores} || 1;
3775     my $maxcpus = $sockets * $cores;
3776
3777     $vcpus = $maxcpus if !$vcpus;
3778
3779     die "you can't add more vcpus than maxcpus\n"
3780         if $vcpus > $maxcpus;
3781
3782     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3783     die "online cpu unplug is not yet possible\n"
3784         if $vcpus < $currentvcpus;
3785
3786     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3787     die "vcpus in running vm is different than configuration\n"
3788         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3789
3790     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3791         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3792     }
3793 }
3794
3795 sub qemu_memory_hotplug {
3796     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3797
3798     return $value if !check_running($vmid);
3799
3800     my $memory = $conf->{memory} || $defaults->{memory};
3801     $value = $defaults->{memory} if !$value;
3802     return $value if $value == $memory;
3803
3804     my $static_memory = $STATICMEM;
3805     my $dimm_memory = $memory - $static_memory;
3806
3807     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3808     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3809
3810
3811     my $sockets = 1;
3812     $sockets = $conf->{sockets} if $conf->{sockets};
3813
3814     if($value > $memory) {
3815
3816         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3817             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3818
3819                 return if $current_size <= $conf->{memory};
3820
3821                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3822                 if (my $err = $@) {
3823                     eval { qemu_objectdel($vmid, "mem-$name"); };
3824                     die $err;
3825                 }
3826
3827                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3828                 if (my $err = $@) {
3829                     eval { qemu_objectdel($vmid, "mem-$name"); };
3830                     die $err;
3831                 }
3832                 #update conf after each succesful module hotplug
3833                 $conf->{memory} = $current_size;
3834                 PVE::QemuConfig->write_config($vmid, $conf);
3835         });
3836
3837     } else {
3838
3839         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3840             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3841
3842                 return if $current_size >= $conf->{memory};
3843                 print "try to unplug memory dimm $name\n";
3844
3845                 my $retry = 0;
3846                 while (1) {
3847                     eval { qemu_devicedel($vmid, $name) };
3848                     sleep 3;
3849                     my $dimm_list = qemu_dimm_list($vmid);
3850                     last if !$dimm_list->{$name};
3851                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3852                     $retry++;
3853                 }
3854
3855                 #update conf after each succesful module unplug
3856                 $conf->{memory} = $current_size;
3857
3858                 eval { qemu_objectdel($vmid, "mem-$name"); };
3859                 PVE::QemuConfig->write_config($vmid, $conf);
3860         });
3861     }
3862 }
3863
3864 sub qemu_dimm_list {
3865     my ($vmid) = @_;
3866
3867     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3868     my $dimms = {};
3869
3870     foreach my $dimm (@$dimmarray) {
3871
3872         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3873         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3874         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3875         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3876         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3877     }
3878     return $dimms;
3879 }
3880
3881 sub qemu_block_set_io_throttle {
3882     my ($vmid, $deviceid,
3883         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3884         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3885
3886     return if !check_running($vmid) ;
3887
3888     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3889         bps => int($bps),
3890         bps_rd => int($bps_rd),
3891         bps_wr => int($bps_wr),
3892         iops => int($iops),
3893         iops_rd => int($iops_rd),
3894         iops_wr => int($iops_wr),
3895         bps_max => int($bps_max),
3896         bps_rd_max => int($bps_rd_max),
3897         bps_wr_max => int($bps_wr_max),
3898         iops_max => int($iops_max),
3899         iops_rd_max => int($iops_rd_max),
3900         iops_wr_max => int($iops_wr_max)
3901     );
3902
3903 }
3904
3905 # old code, only used to shutdown old VM after update
3906 sub __read_avail {
3907     my ($fh, $timeout) = @_;
3908
3909     my $sel = new IO::Select;
3910     $sel->add($fh);
3911
3912     my $res = '';
3913     my $buf;
3914
3915     my @ready;
3916     while (scalar (@ready = $sel->can_read($timeout))) {
3917         my $count;
3918         if ($count = $fh->sysread($buf, 8192)) {
3919             if ($buf =~ /^(.*)\(qemu\) $/s) {
3920                 $res .= $1;
3921                 last;
3922             } else {
3923                 $res .= $buf;
3924             }
3925         } else {
3926             if (!defined($count)) {
3927                 die "$!\n";
3928             }
3929             last;
3930         }
3931     }
3932
3933     die "monitor read timeout\n" if !scalar(@ready);
3934
3935     return $res;
3936 }
3937
3938 # old code, only used to shutdown old VM after update
3939 sub vm_monitor_command {
3940     my ($vmid, $cmdstr, $nocheck) = @_;
3941
3942     my $res;
3943
3944     eval {
3945         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3946
3947         my $sname = "${var_run_tmpdir}/$vmid.mon";
3948
3949         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3950             die "unable to connect to VM $vmid socket - $!\n";
3951
3952         my $timeout = 3;
3953
3954         # hack: migrate sometime blocks the monitor (when migrate_downtime
3955         # is set)
3956         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3957             $timeout = 60*60; # 1 hour
3958         }
3959
3960         # read banner;
3961         my $data = __read_avail($sock, $timeout);
3962
3963         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3964             die "got unexpected qemu monitor banner\n";
3965         }
3966
3967         my $sel = new IO::Select;
3968         $sel->add($sock);
3969
3970         if (!scalar(my @ready = $sel->can_write($timeout))) {
3971             die "monitor write error - timeout";
3972         }
3973
3974         my $fullcmd = "$cmdstr\r";
3975
3976         # syslog('info', "VM $vmid monitor command: $cmdstr");
3977
3978         my $b;
3979         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3980             die "monitor write error - $!";
3981         }
3982
3983         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3984
3985         $timeout = 20;
3986
3987         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3988             $timeout = 60*60; # 1 hour
3989         } elsif ($cmdstr =~ m/^(eject|change)/) {
3990             $timeout = 60; # note: cdrom mount command is slow
3991         }
3992         if ($res = __read_avail($sock, $timeout)) {
3993
3994             my @lines = split("\r?\n", $res);
3995
3996             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3997
3998             $res = join("\n", @lines);
3999             $res .= "\n";
4000         }
4001     };
4002
4003     my $err = $@;
4004
4005     if ($err) {
4006         syslog("err", "VM $vmid monitor command failed - $err");
4007         die $err;
4008     }
4009
4010     return $res;
4011 }
4012
4013 sub qemu_block_resize {
4014     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4015
4016     my $running = check_running($vmid);
4017
4018     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4019
4020     return if !$running;
4021
4022     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4023
4024 }
4025
4026 sub qemu_volume_snapshot {
4027     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4028
4029     my $running = check_running($vmid);
4030
4031     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4032         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4033     } else {
4034         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4035     }
4036 }
4037
4038 sub qemu_volume_snapshot_delete {
4039     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4040
4041     my $running = check_running($vmid);
4042
4043     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4044
4045     return if !$running;
4046
4047     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4048 }
4049
4050 sub set_migration_caps {
4051     my ($vmid) = @_;
4052
4053     my $cap_ref = [];
4054
4055     my $enabled_cap = {
4056         "auto-converge" => 1,
4057         "xbzrle" => 1,
4058         "x-rdma-pin-all" => 0,
4059         "zero-blocks" => 0,
4060         "compress" => 0
4061     };
4062
4063     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4064
4065     for my $supported_capability (@$supported_capabilities) {
4066         push @$cap_ref, {
4067             capability => $supported_capability->{capability},
4068             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4069         };
4070     }
4071
4072     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4073 }
4074
4075 my $fast_plug_option = {
4076     'lock' => 1,
4077     'name' => 1,
4078     'onboot' => 1,
4079     'shares' => 1,
4080     'startup' => 1,
4081     'description' => 1,
4082 };
4083
4084 # hotplug changes in [PENDING]
4085 # $selection hash can be used to only apply specified options, for
4086 # example: { cores => 1 } (only apply changed 'cores')
4087 # $errors ref is used to return error messages
4088 sub vmconfig_hotplug_pending {
4089     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4090
4091     my $defaults = load_defaults();
4092
4093     # commit values which do not have any impact on running VM first
4094     # Note: those option cannot raise errors, we we do not care about
4095     # $selection and always apply them.
4096
4097     my $add_error = sub {
4098         my ($opt, $msg) = @_;
4099         $errors->{$opt} = "hotplug problem - $msg";
4100     };
4101
4102     my $changes = 0;
4103     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4104         if ($fast_plug_option->{$opt}) {
4105             $conf->{$opt} = $conf->{pending}->{$opt};
4106             delete $conf->{pending}->{$opt};
4107             $changes = 1;
4108         }
4109     }
4110
4111     if ($changes) {
4112         PVE::QemuConfig->write_config($vmid, $conf);
4113         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4114     }
4115
4116     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4117
4118     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4119     while (my ($opt, $force) = each %$pending_delete_hash) {
4120         next if $selection && !$selection->{$opt};
4121         eval {
4122             if ($opt eq 'hotplug') {
4123                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4124             } elsif ($opt eq 'tablet') {
4125                 die "skip\n" if !$hotplug_features->{usb};
4126                 if ($defaults->{tablet}) {
4127                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4128                 } else {
4129                     vm_deviceunplug($vmid, $conf, $opt);
4130                 }
4131             } elsif ($opt eq 'vcpus') {
4132                 die "skip\n" if !$hotplug_features->{cpu};
4133                 qemu_cpu_hotplug($vmid, $conf, undef);
4134             } elsif ($opt eq 'balloon') {
4135                 # enable balloon device is not hotpluggable
4136                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4137             } elsif ($fast_plug_option->{$opt}) {
4138                 # do nothing
4139             } elsif ($opt =~ m/^net(\d+)$/) {
4140                 die "skip\n" if !$hotplug_features->{network};
4141                 vm_deviceunplug($vmid, $conf, $opt);
4142             } elsif (is_valid_drivename($opt)) {
4143                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4144                 vm_deviceunplug($vmid, $conf, $opt);
4145                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4146             } elsif ($opt =~ m/^memory$/) {
4147                 die "skip\n" if !$hotplug_features->{memory};
4148                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4149             } elsif ($opt eq 'cpuunits') {
4150                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4151             } elsif ($opt eq 'cpulimit') {
4152                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4153             } else {
4154                 die "skip\n";
4155             }
4156         };
4157         if (my $err = $@) {
4158             &$add_error($opt, $err) if $err ne "skip\n";
4159         } else {
4160             # save new config if hotplug was successful
4161             delete $conf->{$opt};
4162             vmconfig_undelete_pending_option($conf, $opt);
4163             PVE::QemuConfig->write_config($vmid, $conf);
4164             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4165         }
4166     }
4167
4168     foreach my $opt (keys %{$conf->{pending}}) {
4169         next if $selection && !$selection->{$opt};
4170         my $value = $conf->{pending}->{$opt};
4171         eval {
4172             if ($opt eq 'hotplug') {
4173                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4174             } elsif ($opt eq 'tablet') {
4175                 die "skip\n" if !$hotplug_features->{usb};
4176                 if ($value == 1) {
4177                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4178                 } elsif ($value == 0) {
4179                     vm_deviceunplug($vmid, $conf, $opt);
4180                 }
4181             } elsif ($opt eq 'vcpus') {
4182                 die "skip\n" if !$hotplug_features->{cpu};
4183                 qemu_cpu_hotplug($vmid, $conf, $value);
4184             } elsif ($opt eq 'balloon') {
4185                 # enable/disable balloning device is not hotpluggable
4186                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4187                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4188                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4189
4190                 # allow manual ballooning if shares is set to zero
4191                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4192                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4193                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4194                 }
4195             } elsif ($opt =~ m/^net(\d+)$/) {
4196                 # some changes can be done without hotplug
4197                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4198                                     $vmid, $opt, $value);
4199             } elsif (is_valid_drivename($opt)) {
4200                 # some changes can be done without hotplug
4201                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4202                                      $vmid, $opt, $value, 1);
4203             } elsif ($opt =~ m/^memory$/) { #dimms
4204                 die "skip\n" if !$hotplug_features->{memory};
4205                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4206             } elsif ($opt eq 'cpuunits') {
4207                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4208             } elsif ($opt eq 'cpulimit') {
4209                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4210                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4211             } else {
4212                 die "skip\n";  # skip non-hot-pluggable options
4213             }
4214         };
4215         if (my $err = $@) {
4216             &$add_error($opt, $err) if $err ne "skip\n";
4217         } else {
4218             # save new config if hotplug was successful
4219             $conf->{$opt} = $value;
4220             delete $conf->{pending}->{$opt};
4221             PVE::QemuConfig->write_config($vmid, $conf);
4222             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4223         }
4224     }
4225 }
4226
4227 sub try_deallocate_drive {
4228     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4229
4230     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4231         my $volid = $drive->{file};
4232         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4233             my $sid = PVE::Storage::parse_volume_id($volid);
4234             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4235
4236             # check if the disk is really unused
4237             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4238                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4239             PVE::Storage::vdisk_free($storecfg, $volid);
4240             return 1;
4241         } else {
4242             # If vm is not owner of this disk remove from config
4243             return 1;
4244         }
4245     }
4246
4247     return undef;
4248 }
4249
4250 sub vmconfig_delete_or_detach_drive {
4251     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4252
4253     my $drive = parse_drive($opt, $conf->{$opt});
4254
4255     my $rpcenv = PVE::RPCEnvironment::get();
4256     my $authuser = $rpcenv->get_user();
4257
4258     if ($force) {
4259         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4260         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4261     } else {
4262         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4263     }
4264 }
4265
4266 sub vmconfig_apply_pending {
4267     my ($vmid, $conf, $storecfg) = @_;
4268
4269     # cold plug
4270
4271     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4272     while (my ($opt, $force) = each %$pending_delete_hash) {
4273         die "internal error" if $opt =~ m/^unused/;
4274         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4275         if (!defined($conf->{$opt})) {
4276             vmconfig_undelete_pending_option($conf, $opt);
4277             PVE::QemuConfig->write_config($vmid, $conf);
4278         } elsif (is_valid_drivename($opt)) {
4279             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4280             vmconfig_undelete_pending_option($conf, $opt);
4281             delete $conf->{$opt};
4282             PVE::QemuConfig->write_config($vmid, $conf);
4283         } else {
4284             vmconfig_undelete_pending_option($conf, $opt);
4285             delete $conf->{$opt};
4286             PVE::QemuConfig->write_config($vmid, $conf);
4287         }
4288     }
4289
4290     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4291
4292     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4293         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4294
4295         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4296             # skip if nothing changed
4297         } elsif (is_valid_drivename($opt)) {
4298             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4299                 if defined($conf->{$opt});
4300             $conf->{$opt} = $conf->{pending}->{$opt};
4301         } else {
4302             $conf->{$opt} = $conf->{pending}->{$opt};
4303         }
4304
4305         delete $conf->{pending}->{$opt};
4306         PVE::QemuConfig->write_config($vmid, $conf);
4307     }
4308 }
4309
4310 my $safe_num_ne = sub {
4311     my ($a, $b) = @_;
4312
4313     return 0 if !defined($a) && !defined($b);
4314     return 1 if !defined($a);
4315     return 1 if !defined($b);
4316
4317     return $a != $b;
4318 };
4319
4320 my $safe_string_ne = sub {
4321     my ($a, $b) = @_;
4322
4323     return 0 if !defined($a) && !defined($b);
4324     return 1 if !defined($a);
4325     return 1 if !defined($b);
4326
4327     return $a ne $b;
4328 };
4329
4330 sub vmconfig_update_net {
4331     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4332
4333     my $newnet = parse_net($value);
4334
4335     if ($conf->{$opt}) {
4336         my $oldnet = parse_net($conf->{$opt});
4337
4338         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4339             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4340             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4341             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4342
4343             # for non online change, we try to hot-unplug
4344             die "skip\n" if !$hotplug;
4345             vm_deviceunplug($vmid, $conf, $opt);
4346         } else {
4347
4348             die "internal error" if $opt !~ m/net(\d+)/;
4349             my $iface = "tap${vmid}i$1";
4350
4351             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4352                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4353                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4354                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4355                 PVE::Network::tap_unplug($iface);
4356                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4357             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4358                 # Rate can be applied on its own but any change above needs to
4359                 # include the rate in tap_plug since OVS resets everything.
4360                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4361             }
4362
4363             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4364                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4365             }
4366
4367             return 1;
4368         }
4369     }
4370
4371     if ($hotplug) {
4372         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4373     } else {
4374         die "skip\n";
4375     }
4376 }
4377
4378 sub vmconfig_update_disk {
4379     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4380
4381     # fixme: do we need force?
4382
4383     my $drive = parse_drive($opt, $value);
4384
4385     if ($conf->{$opt}) {
4386
4387         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4388
4389             my $media = $drive->{media} || 'disk';
4390             my $oldmedia = $old_drive->{media} || 'disk';
4391             die "unable to change media type\n" if $media ne $oldmedia;
4392
4393             if (!drive_is_cdrom($old_drive)) {
4394
4395                 if ($drive->{file} ne $old_drive->{file}) {
4396
4397                     die "skip\n" if !$hotplug;
4398
4399                     # unplug and register as unused
4400                     vm_deviceunplug($vmid, $conf, $opt);
4401                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4402
4403                 } else {
4404                     # update existing disk
4405
4406                     # skip non hotpluggable value
4407                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4408                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4409                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4410                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4411                         die "skip\n";
4412                     }
4413
4414                     # apply throttle
4415                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4416                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4417                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4418                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4419                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4420                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4421                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4422                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4423                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4424                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4425                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4426                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4427
4428                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4429                                                    ($drive->{mbps} || 0)*1024*1024,
4430                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4431                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4432                                                    $drive->{iops} || 0,
4433                                                    $drive->{iops_rd} || 0,
4434                                                    $drive->{iops_wr} || 0,
4435                                                    ($drive->{mbps_max} || 0)*1024*1024,
4436                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4437                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4438                                                    $drive->{iops_max} || 0,
4439                                                    $drive->{iops_rd_max} || 0,
4440                                                    $drive->{iops_wr_max} || 0);
4441
4442                     }
4443
4444                     return 1;
4445                 }
4446
4447             } else { # cdrom
4448
4449                 if ($drive->{file} eq 'none') {
4450                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4451                 } else {
4452                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4453                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4454                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4455                 }
4456
4457                 return 1;
4458             }
4459         }
4460     }
4461
4462     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4463     # hotplug new disks
4464     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4465     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4466 }
4467
4468 sub vm_start {
4469     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4470         $forcemachine, $spice_ticket) = @_;
4471
4472     PVE::QemuConfig->lock_config($vmid, sub {
4473         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4474
4475         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4476
4477         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4478
4479         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4480
4481         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4482             vmconfig_apply_pending($vmid, $conf, $storecfg);
4483             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4484         }
4485
4486         my $defaults = load_defaults();
4487
4488         # set environment variable useful inside network script
4489         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4490
4491         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4492
4493         my $migrate_port = 0;
4494         my $migrate_uri;
4495         if ($statefile) {
4496             if ($statefile eq 'tcp') {
4497                 my $localip = "localhost";
4498                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4499                 my $nodename = PVE::INotify::nodename();
4500                 if ($datacenterconf->{migration_unsecure}) {
4501                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4502                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4503                 }
4504                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4505                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4506                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4507                 push @$cmd, '-incoming', $migrate_uri;
4508                 push @$cmd, '-S';
4509             } else {
4510                 push @$cmd, '-loadstate', $statefile;
4511             }
4512         } elsif ($paused) {
4513             push @$cmd, '-S';
4514         }
4515
4516         # host pci devices
4517         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4518           my $d = parse_hostpci($conf->{"hostpci$i"});
4519           next if !$d;
4520           my $pcidevices = $d->{pciid};
4521           foreach my $pcidevice (@$pcidevices) {
4522                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4523
4524                 my $info = pci_device_info("0000:$pciid");
4525                 die "IOMMU not present\n" if !check_iommu_support();
4526                 die "no pci device info for device '$pciid'\n" if !$info;
4527                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4528                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4529           }
4530         }
4531
4532         PVE::Storage::activate_volumes($storecfg, $vollist);
4533
4534         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4535                     umask => 0077); };
4536
4537         if (my $err = $@) {
4538             # deactivate volumes if start fails
4539             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4540             die "start failed: $err";
4541         }
4542
4543         print "migration listens on $migrate_uri\n" if $migrate_uri;
4544
4545         if ($statefile && $statefile ne 'tcp')  {
4546             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4547             warn $@ if $@;
4548         }
4549
4550         if ($migratedfrom) {
4551
4552             eval {
4553                 set_migration_caps($vmid);
4554             };
4555             warn $@ if $@;
4556
4557             if ($spice_port) {
4558                 print "spice listens on port $spice_port\n";
4559                 if ($spice_ticket) {
4560                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4561                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4562                 }
4563             }
4564
4565         } else {
4566
4567             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4568                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4569                     if $conf->{balloon};
4570             }
4571
4572             foreach my $opt (keys %$conf) {
4573                 next if $opt !~  m/^net\d+$/;
4574                 my $nicconf = parse_net($conf->{$opt});
4575                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4576             }
4577         }
4578
4579         vm_mon_cmd_nocheck($vmid, 'qom-set',
4580                     path => "machine/peripheral/balloon0",
4581                     property => "guest-stats-polling-interval",
4582                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4583
4584     });
4585 }
4586
4587 sub vm_mon_cmd {
4588     my ($vmid, $execute, %params) = @_;
4589
4590     my $cmd = { execute => $execute, arguments => \%params };
4591     vm_qmp_command($vmid, $cmd);
4592 }
4593
4594 sub vm_mon_cmd_nocheck {
4595     my ($vmid, $execute, %params) = @_;
4596
4597     my $cmd = { execute => $execute, arguments => \%params };
4598     vm_qmp_command($vmid, $cmd, 1);
4599 }
4600
4601 sub vm_qmp_command {
4602     my ($vmid, $cmd, $nocheck) = @_;
4603
4604     my $res;
4605
4606     my $timeout;
4607     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4608         $timeout = $cmd->{arguments}->{timeout};
4609         delete $cmd->{arguments}->{timeout};
4610     }
4611
4612     eval {
4613         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4614         my $sname = qmp_socket($vmid);
4615         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4616             my $qmpclient = PVE::QMPClient->new();
4617
4618             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4619         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4620             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4621                 if scalar(%{$cmd->{arguments}});
4622             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4623         } else {
4624             die "unable to open monitor socket\n";
4625         }
4626     };
4627     if (my $err = $@) {
4628         syslog("err", "VM $vmid qmp command failed - $err");
4629         die $err;
4630     }
4631
4632     return $res;
4633 }
4634
4635 sub vm_human_monitor_command {
4636     my ($vmid, $cmdline) = @_;
4637
4638     my $res;
4639
4640     my $cmd = {
4641         execute => 'human-monitor-command',
4642         arguments => { 'command-line' => $cmdline},
4643     };
4644
4645     return vm_qmp_command($vmid, $cmd);
4646 }
4647
4648 sub vm_commandline {
4649     my ($storecfg, $vmid) = @_;
4650
4651     my $conf = PVE::QemuConfig->load_config($vmid);
4652
4653     my $defaults = load_defaults();
4654
4655     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4656
4657     return join(' ', @$cmd);
4658 }
4659
4660 sub vm_reset {
4661     my ($vmid, $skiplock) = @_;
4662
4663     PVE::QemuConfig->lock_config($vmid, sub {
4664
4665         my $conf = PVE::QemuConfig->load_config($vmid);
4666
4667         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4668
4669         vm_mon_cmd($vmid, "system_reset");
4670     });
4671 }
4672
4673 sub get_vm_volumes {
4674     my ($conf) = @_;
4675
4676     my $vollist = [];
4677     foreach_volid($conf, sub {
4678         my ($volid, $is_cdrom) = @_;
4679
4680         return if $volid =~ m|^/|;
4681
4682         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4683         return if !$sid;
4684
4685         push @$vollist, $volid;
4686     });
4687
4688     return $vollist;
4689 }
4690
4691 sub vm_stop_cleanup {
4692     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4693
4694     eval {
4695
4696         if (!$keepActive) {
4697             my $vollist = get_vm_volumes($conf);
4698             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4699         }
4700
4701         foreach my $ext (qw(mon qmp pid vnc qga)) {
4702             unlink "/var/run/qemu-server/${vmid}.$ext";
4703         }
4704
4705         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4706     };
4707     warn $@ if $@; # avoid errors - just warn
4708 }
4709
4710 # Note: use $nockeck to skip tests if VM configuration file exists.
4711 # We need that when migration VMs to other nodes (files already moved)
4712 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4713 sub vm_stop {
4714     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4715
4716     $force = 1 if !defined($force) && !$shutdown;
4717
4718     if ($migratedfrom){
4719         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4720         kill 15, $pid if $pid;
4721         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4722         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4723         return;
4724     }
4725
4726     PVE::QemuConfig->lock_config($vmid, sub {
4727
4728         my $pid = check_running($vmid, $nocheck);
4729         return if !$pid;
4730
4731         my $conf;
4732         if (!$nocheck) {
4733             $conf = PVE::QemuConfig->load_config($vmid);
4734             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4735             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4736                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4737                 $timeout = $opts->{down} if $opts->{down};
4738             }
4739         }
4740
4741         $timeout = 60 if !defined($timeout);
4742
4743         eval {
4744             if ($shutdown) {
4745                 if (defined($conf) && $conf->{agent}) {
4746                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4747                 } else {
4748                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4749                 }
4750             } else {
4751                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4752             }
4753         };
4754         my $err = $@;
4755
4756         if (!$err) {
4757             my $count = 0;
4758             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4759                 $count++;
4760                 sleep 1;
4761             }
4762
4763             if ($count >= $timeout) {
4764                 if ($force) {
4765                     warn "VM still running - terminating now with SIGTERM\n";
4766                     kill 15, $pid;
4767                 } else {
4768                     die "VM quit/powerdown failed - got timeout\n";
4769                 }
4770             } else {
4771                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4772                 return;
4773             }
4774         } else {
4775             if ($force) {
4776                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4777                 kill 15, $pid;
4778             } else {
4779                 die "VM quit/powerdown failed\n";
4780             }
4781         }
4782
4783         # wait again
4784         $timeout = 10;
4785
4786         my $count = 0;
4787         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4788             $count++;
4789             sleep 1;
4790         }
4791
4792         if ($count >= $timeout) {
4793             warn "VM still running - terminating now with SIGKILL\n";
4794             kill 9, $pid;
4795             sleep 1;
4796         }
4797
4798         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4799    });
4800 }
4801
4802 sub vm_suspend {
4803     my ($vmid, $skiplock) = @_;
4804
4805     PVE::QemuConfig->lock_config($vmid, sub {
4806
4807         my $conf = PVE::QemuConfig->load_config($vmid);
4808
4809         PVE::QemuConfig->check_lock($conf)
4810             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4811
4812         vm_mon_cmd($vmid, "stop");
4813     });
4814 }
4815
4816 sub vm_resume {
4817     my ($vmid, $skiplock, $nocheck) = @_;
4818
4819     PVE::QemuConfig->lock_config($vmid, sub {
4820
4821         if (!$nocheck) {
4822
4823             my $conf = PVE::QemuConfig->load_config($vmid);
4824
4825             PVE::QemuConfig->check_lock($conf)
4826                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4827
4828             vm_mon_cmd($vmid, "cont");
4829
4830         } else {
4831             vm_mon_cmd_nocheck($vmid, "cont");
4832         }
4833     });
4834 }
4835
4836 sub vm_sendkey {
4837     my ($vmid, $skiplock, $key) = @_;
4838
4839     PVE::QemuConfig->lock_config($vmid, sub {
4840
4841         my $conf = PVE::QemuConfig->load_config($vmid);
4842
4843         # there is no qmp command, so we use the human monitor command
4844         vm_human_monitor_command($vmid, "sendkey $key");
4845     });
4846 }
4847
4848 sub vm_destroy {
4849     my ($storecfg, $vmid, $skiplock) = @_;
4850
4851     PVE::QemuConfig->lock_config($vmid, sub {
4852
4853         my $conf = PVE::QemuConfig->load_config($vmid);
4854
4855         if (!check_running($vmid)) {
4856             destroy_vm($storecfg, $vmid, undef, $skiplock);
4857         } else {
4858             die "VM $vmid is running - destroy failed\n";
4859         }
4860     });
4861 }
4862
4863 # pci helpers
4864
4865 sub file_write {
4866     my ($filename, $buf) = @_;
4867
4868     my $fh = IO::File->new($filename, "w");
4869     return undef if !$fh;
4870
4871     my $res = print $fh $buf;
4872
4873     $fh->close();
4874
4875     return $res;
4876 }
4877
4878 sub pci_device_info {
4879     my ($name) = @_;
4880
4881     my $res;
4882
4883     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4884     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4885
4886     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4887     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4888
4889     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4890     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4891
4892     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4893     return undef if !defined($product) || $product !~ s/^0x//;
4894
4895     $res = {
4896         name => $name,
4897         vendor => $vendor,
4898         product => $product,
4899         domain => $domain,
4900         bus => $bus,
4901         slot => $slot,
4902         func => $func,
4903         irq => $irq,
4904         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4905     };
4906
4907     return $res;
4908 }
4909
4910 sub pci_dev_reset {
4911     my ($dev) = @_;
4912
4913     my $name = $dev->{name};
4914
4915     my $fn = "$pcisysfs/devices/$name/reset";
4916
4917     return file_write($fn, "1");
4918 }
4919
4920 sub pci_dev_bind_to_vfio {
4921     my ($dev) = @_;
4922
4923     my $name = $dev->{name};
4924
4925     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4926
4927     if (!-d $vfio_basedir) {
4928         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4929     }
4930     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4931
4932     my $testdir = "$vfio_basedir/$name";
4933     return 1 if -d $testdir;
4934
4935     my $data = "$dev->{vendor} $dev->{product}";
4936     return undef if !file_write("$vfio_basedir/new_id", $data);
4937
4938     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4939     if (!file_write($fn, $name)) {
4940         return undef if -f $fn;
4941     }
4942
4943     $fn = "$vfio_basedir/bind";
4944     if (! -d $testdir) {
4945         return undef if !file_write($fn, $name);
4946     }
4947
4948     return -d $testdir;
4949 }
4950
4951 sub pci_dev_group_bind_to_vfio {
4952     my ($pciid) = @_;
4953
4954     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4955
4956     if (!-d $vfio_basedir) {
4957         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4958     }
4959     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4960
4961     # get IOMMU group devices
4962     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4963       my @devs = grep /^0000:/, readdir($D);
4964     closedir($D);
4965
4966     foreach my $pciid (@devs) {
4967         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4968
4969         # pci bridges, switches or root ports are not supported
4970         # they have a pci_bus subdirectory so skip them
4971         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4972
4973         my $info = pci_device_info($1);
4974         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4975     }
4976
4977     return 1;
4978 }
4979
4980 sub print_pci_addr {
4981     my ($id, $bridges) = @_;
4982
4983     my $res = '';
4984     my $devices = {
4985         piix3 => { bus => 0, addr => 1 },
4986         #addr2 : first videocard
4987         balloon0 => { bus => 0, addr => 3 },
4988         watchdog => { bus => 0, addr => 4 },
4989         scsihw0 => { bus => 0, addr => 5 },
4990         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4991         scsihw1 => { bus => 0, addr => 6 },
4992         ahci0 => { bus => 0, addr => 7 },
4993         qga0 => { bus => 0, addr => 8 },
4994         spice => { bus => 0, addr => 9 },
4995         virtio0 => { bus => 0, addr => 10 },
4996         virtio1 => { bus => 0, addr => 11 },
4997         virtio2 => { bus => 0, addr => 12 },
4998         virtio3 => { bus => 0, addr => 13 },
4999         virtio4 => { bus => 0, addr => 14 },
5000         virtio5 => { bus => 0, addr => 15 },
5001         hostpci0 => { bus => 0, addr => 16 },
5002         hostpci1 => { bus => 0, addr => 17 },
5003         net0 => { bus => 0, addr => 18 },
5004         net1 => { bus => 0, addr => 19 },
5005         net2 => { bus => 0, addr => 20 },
5006         net3 => { bus => 0, addr => 21 },
5007         net4 => { bus => 0, addr => 22 },
5008         net5 => { bus => 0, addr => 23 },
5009         vga1 => { bus => 0, addr => 24 },
5010         vga2 => { bus => 0, addr => 25 },
5011         vga3 => { bus => 0, addr => 26 },
5012         hostpci2 => { bus => 0, addr => 27 },
5013         hostpci3 => { bus => 0, addr => 28 },
5014         #addr29 : usb-host (pve-usb.cfg)
5015         'pci.1' => { bus => 0, addr => 30 },
5016         'pci.2' => { bus => 0, addr => 31 },
5017         'net6' => { bus => 1, addr => 1 },
5018         'net7' => { bus => 1, addr => 2 },
5019         'net8' => { bus => 1, addr => 3 },
5020         'net9' => { bus => 1, addr => 4 },
5021         'net10' => { bus => 1, addr => 5 },
5022         'net11' => { bus => 1, addr => 6 },
5023         'net12' => { bus => 1, addr => 7 },
5024         'net13' => { bus => 1, addr => 8 },
5025         'net14' => { bus => 1, addr => 9 },
5026         'net15' => { bus => 1, addr => 10 },
5027         'net16' => { bus => 1, addr => 11 },
5028         'net17' => { bus => 1, addr => 12 },
5029         'net18' => { bus => 1, addr => 13 },
5030         'net19' => { bus => 1, addr => 14 },
5031         'net20' => { bus => 1, addr => 15 },
5032         'net21' => { bus => 1, addr => 16 },
5033         'net22' => { bus => 1, addr => 17 },
5034         'net23' => { bus => 1, addr => 18 },
5035         'net24' => { bus => 1, addr => 19 },
5036         'net25' => { bus => 1, addr => 20 },
5037         'net26' => { bus => 1, addr => 21 },
5038         'net27' => { bus => 1, addr => 22 },
5039         'net28' => { bus => 1, addr => 23 },
5040         'net29' => { bus => 1, addr => 24 },
5041         'net30' => { bus => 1, addr => 25 },
5042         'net31' => { bus => 1, addr => 26 },
5043         'xhci' => { bus => 1, addr => 27 },
5044         'virtio6' => { bus => 2, addr => 1 },
5045         'virtio7' => { bus => 2, addr => 2 },
5046         'virtio8' => { bus => 2, addr => 3 },
5047         'virtio9' => { bus => 2, addr => 4 },
5048         'virtio10' => { bus => 2, addr => 5 },
5049         'virtio11' => { bus => 2, addr => 6 },
5050         'virtio12' => { bus => 2, addr => 7 },
5051         'virtio13' => { bus => 2, addr => 8 },
5052         'virtio14' => { bus => 2, addr => 9 },
5053         'virtio15' => { bus => 2, addr => 10 },
5054         'virtioscsi0' => { bus => 3, addr => 1 },
5055         'virtioscsi1' => { bus => 3, addr => 2 },
5056         'virtioscsi2' => { bus => 3, addr => 3 },
5057         'virtioscsi3' => { bus => 3, addr => 4 },
5058         'virtioscsi4' => { bus => 3, addr => 5 },
5059         'virtioscsi5' => { bus => 3, addr => 6 },
5060         'virtioscsi6' => { bus => 3, addr => 7 },
5061         'virtioscsi7' => { bus => 3, addr => 8 },
5062         'virtioscsi8' => { bus => 3, addr => 9 },
5063         'virtioscsi9' => { bus => 3, addr => 10 },
5064         'virtioscsi10' => { bus => 3, addr => 11 },
5065         'virtioscsi11' => { bus => 3, addr => 12 },
5066         'virtioscsi12' => { bus => 3, addr => 13 },
5067         'virtioscsi13' => { bus => 3, addr => 14 },
5068         'virtioscsi14' => { bus => 3, addr => 15 },
5069         'virtioscsi15' => { bus => 3, addr => 16 },
5070         'virtioscsi16' => { bus => 3, addr => 17 },
5071         'virtioscsi17' => { bus => 3, addr => 18 },
5072         'virtioscsi18' => { bus => 3, addr => 19 },
5073         'virtioscsi19' => { bus => 3, addr => 20 },
5074         'virtioscsi20' => { bus => 3, addr => 21 },
5075         'virtioscsi21' => { bus => 3, addr => 22 },
5076         'virtioscsi22' => { bus => 3, addr => 23 },
5077         'virtioscsi23' => { bus => 3, addr => 24 },
5078         'virtioscsi24' => { bus => 3, addr => 25 },
5079         'virtioscsi25' => { bus => 3, addr => 26 },
5080         'virtioscsi26' => { bus => 3, addr => 27 },
5081         'virtioscsi27' => { bus => 3, addr => 28 },
5082         'virtioscsi28' => { bus => 3, addr => 29 },
5083         'virtioscsi29' => { bus => 3, addr => 30 },
5084         'virtioscsi30' => { bus => 3, addr => 31 },
5085
5086     };
5087
5088     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5089            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5090            my $bus = $devices->{$id}->{bus};
5091            $res = ",bus=pci.$bus,addr=$addr";
5092            $bridges->{$bus} = 1 if $bridges;
5093     }
5094     return $res;
5095
5096 }
5097
5098 sub print_pcie_addr {
5099     my ($id) = @_;
5100
5101     my $res = '';
5102     my $devices = {
5103         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5104         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5105         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5106         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5107     };
5108
5109     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5110            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5111            my $bus = $devices->{$id}->{bus};
5112            $res = ",bus=$bus,addr=$addr";
5113     }
5114     return $res;
5115
5116 }
5117
5118 # vzdump restore implementaion
5119
5120 sub tar_archive_read_firstfile {
5121     my $archive = shift;
5122
5123     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5124
5125     # try to detect archive type first
5126     my $pid = open (TMP, "tar tf '$archive'|") ||
5127         die "unable to open file '$archive'\n";
5128     my $firstfile = <TMP>;
5129     kill 15, $pid;
5130     close TMP;
5131
5132     die "ERROR: archive contaions no data\n" if !$firstfile;
5133     chomp $firstfile;
5134
5135     return $firstfile;
5136 }
5137
5138 sub tar_restore_cleanup {
5139     my ($storecfg, $statfile) = @_;
5140
5141     print STDERR "starting cleanup\n";
5142
5143     if (my $fd = IO::File->new($statfile, "r")) {
5144         while (defined(my $line = <$fd>)) {
5145             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5146                 my $volid = $2;
5147                 eval {
5148                     if ($volid =~ m|^/|) {
5149                         unlink $volid || die 'unlink failed\n';
5150                     } else {
5151                         PVE::Storage::vdisk_free($storecfg, $volid);
5152                     }
5153                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5154                 };
5155                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5156             } else {
5157                 print STDERR "unable to parse line in statfile - $line";
5158             }
5159         }
5160         $fd->close();
5161     }
5162 }
5163
5164 sub restore_archive {
5165     my ($archive, $vmid, $user, $opts) = @_;
5166
5167     my $format = $opts->{format};
5168     my $comp;
5169
5170     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5171         $format = 'tar' if !$format;
5172         $comp = 'gzip';
5173     } elsif ($archive =~ m/\.tar$/) {
5174         $format = 'tar' if !$format;
5175     } elsif ($archive =~ m/.tar.lzo$/) {
5176         $format = 'tar' if !$format;
5177         $comp = 'lzop';
5178     } elsif ($archive =~ m/\.vma$/) {
5179         $format = 'vma' if !$format;
5180     } elsif ($archive =~ m/\.vma\.gz$/) {
5181         $format = 'vma' if !$format;
5182         $comp = 'gzip';
5183     } elsif ($archive =~ m/\.vma\.lzo$/) {
5184         $format = 'vma' if !$format;
5185         $comp = 'lzop';
5186     } else {
5187         $format = 'vma' if !$format; # default
5188     }
5189
5190     # try to detect archive format
5191     if ($format eq 'tar') {
5192         return restore_tar_archive($archive, $vmid, $user, $opts);
5193     } else {
5194         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5195     }
5196 }
5197
5198 sub restore_update_config_line {
5199     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5200
5201     return if $line =~ m/^\#qmdump\#/;
5202     return if $line =~ m/^\#vzdump\#/;
5203     return if $line =~ m/^lock:/;
5204     return if $line =~ m/^unused\d+:/;
5205     return if $line =~ m/^parent:/;
5206     return if $line =~ m/^template:/; # restored VM is never a template
5207
5208     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5209         # try to convert old 1.X settings
5210         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5211         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5212             my ($model, $macaddr) = split(/\=/, $devconfig);
5213             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5214             my $net = {
5215                 model => $model,
5216                 bridge => "vmbr$ind",
5217                 macaddr => $macaddr,
5218             };
5219             my $netstr = print_net($net);
5220
5221             print $outfd "net$cookie->{netcount}: $netstr\n";
5222             $cookie->{netcount}++;
5223         }
5224     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5225         my ($id, $netstr) = ($1, $2);
5226         my $net = parse_net($netstr);
5227         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5228         $netstr = print_net($net);
5229         print $outfd "$id: $netstr\n";
5230     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5231         my $virtdev = $1;
5232         my $value = $3;
5233         my $di = parse_drive($virtdev, $value);
5234         if (defined($di->{backup}) && !$di->{backup}) {
5235             print $outfd "#$line";
5236         } elsif ($map->{$virtdev}) {
5237             delete $di->{format}; # format can change on restore
5238             $di->{file} = $map->{$virtdev};
5239             $value = print_drive($vmid, $di);
5240             print $outfd "$virtdev: $value\n";
5241         } else {
5242             print $outfd $line;
5243         }
5244     } else {
5245         print $outfd $line;
5246     }
5247 }
5248
5249 sub scan_volids {
5250     my ($cfg, $vmid) = @_;
5251
5252     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5253
5254     my $volid_hash = {};
5255     foreach my $storeid (keys %$info) {
5256         foreach my $item (@{$info->{$storeid}}) {
5257             next if !($item->{volid} && $item->{size});
5258             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5259             $volid_hash->{$item->{volid}} = $item;
5260         }
5261     }
5262
5263     return $volid_hash;
5264 }
5265
5266 sub is_volume_in_use {
5267     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5268
5269     my $path = PVE::Storage::path($storecfg, $volid);
5270
5271     my $scan_config = sub {
5272         my ($cref, $snapname) = @_;
5273
5274         foreach my $key (keys %$cref) {
5275             my $value = $cref->{$key};
5276             if (is_valid_drivename($key)) {
5277                 next if $skip_drive && $key eq $skip_drive;
5278                 my $drive = parse_drive($key, $value);
5279                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5280                 return 1 if $volid eq $drive->{file};
5281                 if ($drive->{file} =~ m!^/!) {
5282                     return 1 if $drive->{file} eq $path;
5283                 } else {
5284                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5285                     next if !$storeid;
5286                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5287                     next if !$scfg;
5288                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5289                 }
5290             }
5291         }
5292
5293         return 0;
5294     };
5295
5296     return 1 if &$scan_config($conf);
5297
5298     undef $skip_drive;
5299
5300     foreach my $snapname (keys %{$conf->{snapshots}}) {
5301         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5302     }
5303
5304     return 0;
5305 }
5306
5307 sub update_disksize {
5308     my ($vmid, $conf, $volid_hash) = @_;
5309
5310     my $changes;
5311
5312     my $used = {};
5313
5314     # Note: it is allowed to define multiple storages with same path (alias), so
5315     # we need to check both 'volid' and real 'path' (two different volid can point
5316     # to the same path).
5317
5318     my $usedpath = {};
5319
5320     # update size info
5321     foreach my $opt (keys %$conf) {
5322         if (is_valid_drivename($opt)) {
5323             my $drive = parse_drive($opt, $conf->{$opt});
5324             my $volid = $drive->{file};
5325             next if !$volid;
5326
5327             $used->{$volid} = 1;
5328             if ($volid_hash->{$volid} &&
5329                 (my $path = $volid_hash->{$volid}->{path})) {
5330                 $usedpath->{$path} = 1;
5331             }
5332
5333             next if drive_is_cdrom($drive);
5334             next if !$volid_hash->{$volid};
5335
5336             $drive->{size} = $volid_hash->{$volid}->{size};
5337             my $new = print_drive($vmid, $drive);
5338             if ($new ne $conf->{$opt}) {
5339                 $changes = 1;
5340                 $conf->{$opt} = $new;
5341             }
5342         }
5343     }
5344
5345     # remove 'unusedX' entry if volume is used
5346     foreach my $opt (keys %$conf) {
5347         next if $opt !~ m/^unused\d+$/;
5348         my $volid = $conf->{$opt};
5349         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5350         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5351             $changes = 1;
5352             delete $conf->{$opt};
5353         }
5354     }
5355
5356     foreach my $volid (sort keys %$volid_hash) {
5357         next if $volid =~ m/vm-$vmid-state-/;
5358         next if $used->{$volid};
5359         my $path = $volid_hash->{$volid}->{path};
5360         next if !$path; # just to be sure
5361         next if $usedpath->{$path};
5362         $changes = 1;
5363         PVE::QemuConfig->add_unused_volume($conf, $volid);
5364         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5365     }
5366
5367     return $changes;
5368 }
5369
5370 sub rescan {
5371     my ($vmid, $nolock) = @_;
5372
5373     my $cfg = PVE::Storage::config();
5374
5375     my $volid_hash = scan_volids($cfg, $vmid);
5376
5377     my $updatefn =  sub {
5378         my ($vmid) = @_;
5379
5380         my $conf = PVE::QemuConfig->load_config($vmid);
5381
5382         PVE::QemuConfig->check_lock($conf);
5383
5384         my $vm_volids = {};
5385         foreach my $volid (keys %$volid_hash) {
5386             my $info = $volid_hash->{$volid};
5387             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5388         }
5389
5390         my $changes = update_disksize($vmid, $conf, $vm_volids);
5391
5392         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5393     };
5394
5395     if (defined($vmid)) {
5396         if ($nolock) {
5397             &$updatefn($vmid);
5398         } else {
5399             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5400         }
5401     } else {
5402         my $vmlist = config_list();
5403         foreach my $vmid (keys %$vmlist) {
5404             if ($nolock) {
5405                 &$updatefn($vmid);
5406             } else {
5407                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5408             }
5409         }
5410     }
5411 }
5412
5413 sub restore_vma_archive {
5414     my ($archive, $vmid, $user, $opts, $comp) = @_;
5415
5416     my $input = $archive eq '-' ? "<&STDIN" : undef;
5417     my $readfrom = $archive;
5418
5419     my $uncomp = '';
5420     if ($comp) {
5421         $readfrom = '-';
5422         my $qarchive = PVE::Tools::shellquote($archive);
5423         if ($comp eq 'gzip') {
5424             $uncomp = "zcat $qarchive|";
5425         } elsif ($comp eq 'lzop') {
5426             $uncomp = "lzop -d -c $qarchive|";
5427         } else {
5428             die "unknown compression method '$comp'\n";
5429         }
5430
5431     }
5432
5433     my $tmpdir = "/var/tmp/vzdumptmp$$";
5434     rmtree $tmpdir;
5435
5436     # disable interrupts (always do cleanups)
5437     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5438         warn "got interrupt - ignored\n";
5439     };
5440
5441     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5442     POSIX::mkfifo($mapfifo, 0600);
5443     my $fifofh;
5444
5445     my $openfifo = sub {
5446         open($fifofh, '>', $mapfifo) || die $!;
5447     };
5448
5449     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5450
5451     my $oldtimeout;
5452     my $timeout = 5;
5453
5454     my $devinfo = {};
5455
5456     my $rpcenv = PVE::RPCEnvironment::get();
5457
5458     my $conffile = PVE::QemuConfig->config_file($vmid);
5459     my $tmpfn = "$conffile.$$.tmp";
5460
5461     # Note: $oldconf is undef if VM does not exists
5462     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5463     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5464
5465     my $print_devmap = sub {
5466         my $virtdev_hash = {};
5467
5468         my $cfgfn = "$tmpdir/qemu-server.conf";
5469
5470         # we can read the config - that is already extracted
5471         my $fh = IO::File->new($cfgfn, "r") ||
5472             "unable to read qemu-server.conf - $!\n";
5473
5474         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5475         if (-f $fwcfgfn) {
5476             my $pve_firewall_dir = '/etc/pve/firewall';
5477             mkdir $pve_firewall_dir; # make sure the dir exists
5478             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5479         }
5480
5481         while (defined(my $line = <$fh>)) {
5482             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5483                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5484                 die "archive does not contain data for drive '$virtdev'\n"
5485                     if !$devinfo->{$devname};
5486                 if (defined($opts->{storage})) {
5487                     $storeid = $opts->{storage} || 'local';
5488                 } elsif (!$storeid) {
5489                     $storeid = 'local';
5490                 }
5491                 $format = 'raw' if !$format;
5492                 $devinfo->{$devname}->{devname} = $devname;
5493                 $devinfo->{$devname}->{virtdev} = $virtdev;
5494                 $devinfo->{$devname}->{format} = $format;
5495                 $devinfo->{$devname}->{storeid} = $storeid;
5496
5497                 # check permission on storage
5498                 my $pool = $opts->{pool}; # todo: do we need that?
5499                 if ($user ne 'root@pam') {
5500                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5501                 }
5502
5503                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5504             }
5505         }
5506
5507         foreach my $devname (keys %$devinfo) {
5508             die "found no device mapping information for device '$devname'\n"
5509                 if !$devinfo->{$devname}->{virtdev};
5510         }
5511
5512         my $cfg = PVE::Storage::config();
5513
5514         # create empty/temp config
5515         if ($oldconf) {
5516             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5517             foreach_drive($oldconf, sub {
5518                 my ($ds, $drive) = @_;
5519
5520                 return if drive_is_cdrom($drive);
5521
5522                 my $volid = $drive->{file};
5523
5524                 return if !$volid || $volid =~ m|^/|;
5525
5526                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5527                 return if !$path || !$owner || ($owner != $vmid);
5528
5529                 # Note: only delete disk we want to restore
5530                 # other volumes will become unused
5531                 if ($virtdev_hash->{$ds}) {
5532                     PVE::Storage::vdisk_free($cfg, $volid);
5533                 }
5534             });
5535
5536             # delete vmstate files
5537             # since after the restore we have no snapshots anymore
5538             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5539                 my $snap = $oldconf->{snapshots}->{$snapname};
5540                 if ($snap->{vmstate}) {
5541                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5542                     if (my $err = $@) {
5543                         warn $err;
5544                     }
5545                 }
5546             }
5547         }
5548
5549         my $map = {};
5550         foreach my $virtdev (sort keys %$virtdev_hash) {
5551             my $d = $virtdev_hash->{$virtdev};
5552             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5553             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5554
5555             # test if requested format is supported
5556             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5557             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5558             $d->{format} = $defFormat if !$supported;
5559
5560             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5561                                                   $d->{format}, undef, $alloc_size);
5562             print STDERR "new volume ID is '$volid'\n";
5563             $d->{volid} = $volid;
5564             my $path = PVE::Storage::path($cfg, $volid);
5565
5566             PVE::Storage::activate_volumes($cfg,[$volid]);
5567
5568             my $write_zeros = 1;
5569             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5570                 $write_zeros = 0;
5571             }
5572
5573             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5574
5575             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5576             $map->{$virtdev} = $volid;
5577         }
5578
5579         $fh->seek(0, 0) || die "seek failed - $!\n";
5580
5581         my $outfd = new IO::File ($tmpfn, "w") ||
5582             die "unable to write config for VM $vmid\n";
5583
5584         my $cookie = { netcount => 0 };
5585         while (defined(my $line = <$fh>)) {
5586             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5587         }
5588
5589         $fh->close();
5590         $outfd->close();
5591     };
5592
5593     eval {
5594         # enable interrupts
5595         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5596             die "interrupted by signal\n";
5597         };
5598         local $SIG{ALRM} = sub { die "got timeout\n"; };
5599
5600         $oldtimeout = alarm($timeout);
5601
5602         my $parser = sub {
5603             my $line = shift;
5604
5605             print "$line\n";
5606
5607             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5608                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5609                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5610             } elsif ($line =~ m/^CTIME: /) {
5611                 # we correctly received the vma config, so we can disable
5612                 # the timeout now for disk allocation (set to 10 minutes, so
5613                 # that we always timeout if something goes wrong)
5614                 alarm(600);
5615                 &$print_devmap();
5616                 print $fifofh "done\n";
5617                 my $tmp = $oldtimeout || 0;
5618                 $oldtimeout = undef;
5619                 alarm($tmp);
5620                 close($fifofh);
5621             }
5622         };
5623
5624         print "restore vma archive: $cmd\n";
5625         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5626     };
5627     my $err = $@;
5628
5629     alarm($oldtimeout) if $oldtimeout;
5630
5631     my $vollist = [];
5632     foreach my $devname (keys %$devinfo) {
5633         my $volid = $devinfo->{$devname}->{volid};
5634         push @$vollist, $volid if $volid;
5635     }
5636
5637     my $cfg = PVE::Storage::config();
5638     PVE::Storage::deactivate_volumes($cfg, $vollist);
5639
5640     unlink $mapfifo;
5641
5642     if ($err) {
5643         rmtree $tmpdir;
5644         unlink $tmpfn;
5645
5646         foreach my $devname (keys %$devinfo) {
5647             my $volid = $devinfo->{$devname}->{volid};
5648             next if !$volid;
5649             eval {
5650                 if ($volid =~ m|^/|) {
5651                     unlink $volid || die 'unlink failed\n';
5652                 } else {
5653                     PVE::Storage::vdisk_free($cfg, $volid);
5654                 }
5655                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5656             };
5657             print STDERR "unable to cleanup '$volid' - $@" if $@;
5658         }
5659         die $err;
5660     }
5661
5662     rmtree $tmpdir;
5663
5664     rename($tmpfn, $conffile) ||
5665         die "unable to commit configuration file '$conffile'\n";
5666
5667     PVE::Cluster::cfs_update(); # make sure we read new file
5668
5669     eval { rescan($vmid, 1); };
5670     warn $@ if $@;
5671 }
5672
5673 sub restore_tar_archive {
5674     my ($archive, $vmid, $user, $opts) = @_;
5675
5676     if ($archive ne '-') {
5677         my $firstfile = tar_archive_read_firstfile($archive);
5678         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5679             if $firstfile ne 'qemu-server.conf';
5680     }
5681
5682     my $storecfg = PVE::Storage::config();
5683
5684     # destroy existing data - keep empty config
5685     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5686     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5687
5688     my $tocmd = "/usr/lib/qemu-server/qmextract";
5689
5690     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5691     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5692     $tocmd .= ' --prealloc' if $opts->{prealloc};
5693     $tocmd .= ' --info' if $opts->{info};
5694
5695     # tar option "xf" does not autodetect compression when read from STDIN,
5696     # so we pipe to zcat
5697     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5698         PVE::Tools::shellquote("--to-command=$tocmd");
5699
5700     my $tmpdir = "/var/tmp/vzdumptmp$$";
5701     mkpath $tmpdir;
5702
5703     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5704     local $ENV{VZDUMP_VMID} = $vmid;
5705     local $ENV{VZDUMP_USER} = $user;
5706
5707     my $conffile = PVE::QemuConfig->config_file($vmid);
5708     my $tmpfn = "$conffile.$$.tmp";
5709
5710     # disable interrupts (always do cleanups)
5711     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5712         print STDERR "got interrupt - ignored\n";
5713     };
5714
5715     eval {
5716         # enable interrupts
5717         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5718             die "interrupted by signal\n";
5719         };
5720
5721         if ($archive eq '-') {
5722             print "extracting archive from STDIN\n";
5723             run_command($cmd, input => "<&STDIN");
5724         } else {
5725             print "extracting archive '$archive'\n";
5726             run_command($cmd);
5727         }
5728
5729         return if $opts->{info};
5730
5731         # read new mapping
5732         my $map = {};
5733         my $statfile = "$tmpdir/qmrestore.stat";
5734         if (my $fd = IO::File->new($statfile, "r")) {
5735             while (defined (my $line = <$fd>)) {
5736                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5737                     $map->{$1} = $2 if $1;
5738                 } else {
5739                     print STDERR "unable to parse line in statfile - $line\n";
5740                 }
5741             }
5742             $fd->close();
5743         }
5744
5745         my $confsrc = "$tmpdir/qemu-server.conf";
5746
5747         my $srcfd = new IO::File($confsrc, "r") ||
5748             die "unable to open file '$confsrc'\n";
5749
5750         my $outfd = new IO::File ($tmpfn, "w") ||
5751             die "unable to write config for VM $vmid\n";
5752
5753         my $cookie = { netcount => 0 };
5754         while (defined (my $line = <$srcfd>)) {
5755             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5756         }
5757
5758         $srcfd->close();
5759         $outfd->close();
5760     };
5761     my $err = $@;
5762
5763     if ($err) {
5764
5765         unlink $tmpfn;
5766
5767         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5768
5769         die $err;
5770     }
5771
5772     rmtree $tmpdir;
5773
5774     rename $tmpfn, $conffile ||
5775         die "unable to commit configuration file '$conffile'\n";
5776
5777     PVE::Cluster::cfs_update(); # make sure we read new file
5778
5779     eval { rescan($vmid, 1); };
5780     warn $@ if $@;
5781 };
5782
5783 sub foreach_writable_storage {
5784     my ($conf, $func) = @_;
5785
5786     my $sidhash = {};
5787
5788     foreach my $ds (keys %$conf) {
5789         next if !is_valid_drivename($ds);
5790
5791         my $drive = parse_drive($ds, $conf->{$ds});
5792         next if !$drive;
5793         next if drive_is_cdrom($drive);
5794
5795         my $volid = $drive->{file};
5796
5797         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5798         $sidhash->{$sid} = $sid if $sid;
5799     }
5800
5801     foreach my $sid (sort keys %$sidhash) {
5802         &$func($sid);
5803     }
5804 }
5805
5806 sub do_snapshots_with_qemu {
5807     my ($storecfg, $volid) = @_;
5808
5809     my $storage_name = PVE::Storage::parse_volume_id($volid);
5810
5811     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5812         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5813         return 1;
5814     }
5815
5816     if ($volid =~ m/\.(qcow2|qed)$/){
5817         return 1;
5818     }
5819
5820     return undef;
5821 }
5822
5823 sub qga_check_running {
5824     my ($vmid) = @_;
5825
5826     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5827     if ($@) {
5828         warn "Qemu Guest Agent are not running - $@";
5829         return 0;
5830     }
5831     return 1;
5832 }
5833
5834 sub template_create {
5835     my ($vmid, $conf, $disk) = @_;
5836
5837     my $storecfg = PVE::Storage::config();
5838
5839     foreach_drive($conf, sub {
5840         my ($ds, $drive) = @_;
5841
5842         return if drive_is_cdrom($drive);
5843         return if $disk && $ds ne $disk;
5844
5845         my $volid = $drive->{file};
5846         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5847
5848         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5849         $drive->{file} = $voliddst;
5850         $conf->{$ds} = print_drive($vmid, $drive);
5851         PVE::QemuConfig->write_config($vmid, $conf);
5852     });
5853 }
5854
5855 sub qemu_img_convert {
5856     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5857
5858     my $storecfg = PVE::Storage::config();
5859     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5860     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5861
5862     if ($src_storeid && $dst_storeid) {
5863
5864         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5865
5866         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5867         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5868
5869         my $src_format = qemu_img_format($src_scfg, $src_volname);
5870         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5871
5872         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5873         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5874
5875         my $cmd = [];
5876         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5877         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5878         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5879         if ($is_zero_initialized) {
5880             push @$cmd, "zeroinit:$dst_path";
5881         } else {
5882             push @$cmd, $dst_path;
5883         }
5884
5885         my $parser = sub {
5886             my $line = shift;
5887             if($line =~ m/\((\S+)\/100\%\)/){
5888                 my $percent = $1;
5889                 my $transferred = int($size * $percent / 100);
5890                 my $remaining = $size - $transferred;
5891
5892                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5893             }
5894
5895         };
5896
5897         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5898         my $err = $@;
5899         die "copy failed: $err" if $err;
5900     }
5901 }
5902
5903 sub qemu_img_format {
5904     my ($scfg, $volname) = @_;
5905
5906     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5907         return $1;
5908     } else {
5909         return "raw";
5910     }
5911 }
5912
5913 sub qemu_drive_mirror {
5914     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5915
5916     my $storecfg = PVE::Storage::config();
5917     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5918
5919     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5920
5921     my $format = qemu_img_format($dst_scfg, $dst_volname);
5922
5923     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5924
5925     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5926
5927     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5928     $opts->{format} = $format if $format;
5929
5930     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5931
5932     eval {
5933     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5934         while (1) {
5935             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5936             my $stat = @$stats[0];
5937             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5938             die "error job is not mirroring" if $stat->{type} ne "mirror";
5939
5940             my $busy = $stat->{busy};
5941             my $ready = $stat->{ready};
5942
5943             if (my $total = $stat->{len}) {
5944                 my $transferred = $stat->{offset} || 0;
5945                 my $remaining = $total - $transferred;
5946                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5947
5948                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5949             }
5950
5951
5952             if ($stat->{ready} eq 'true') {
5953
5954                 last if $vmiddst != $vmid;
5955
5956                 # try to switch the disk if source and destination are on the same guest
5957                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5958                 last if !$@;
5959                 die $@ if $@ !~ m/cannot be completed/;
5960             }
5961             sleep 1;
5962         }
5963
5964
5965     };
5966     my $err = $@;
5967
5968     my $cancel_job = sub {
5969         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5970         while (1) {
5971             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5972             my $stat = @$stats[0];
5973             last if !$stat;
5974             sleep 1;
5975         }
5976     };
5977
5978     if ($err) {
5979         eval { &$cancel_job(); };
5980         die "mirroring error: $err";
5981     }
5982
5983     if ($vmiddst != $vmid) {
5984         # if we clone a disk for a new target vm, we don't switch the disk
5985         &$cancel_job(); # so we call block-job-cancel
5986     }
5987 }
5988
5989 sub clone_disk {
5990     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5991         $newvmid, $storage, $format, $full, $newvollist) = @_;
5992
5993     my $newvolid;
5994
5995     if (!$full) {
5996         print "create linked clone of drive $drivename ($drive->{file})\n";
5997         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5998         push @$newvollist, $newvolid;
5999     } else {
6000         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6001         $storeid = $storage if $storage;
6002
6003         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6004         if (!$format) {
6005             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6006             $format = qemu_img_format($scfg, $volname);
6007         }
6008
6009         # test if requested format is supported - else use default
6010         my $supported = grep { $_ eq $format } @$validFormats;
6011         $format = $defFormat if !$supported;
6012
6013         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6014
6015         print "create full clone of drive $drivename ($drive->{file})\n";
6016         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6017         push @$newvollist, $newvolid;
6018
6019         PVE::Storage::activate_volumes($storecfg, $newvollist);
6020
6021         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6022         if (!$running || $snapname) {
6023             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6024         } else {
6025             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
6026         }
6027     }
6028
6029     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6030
6031     my $disk = $drive;
6032     $disk->{format} = undef;
6033     $disk->{file} = $newvolid;
6034     $disk->{size} = $size;
6035
6036     return $disk;
6037 }
6038
6039 # this only works if VM is running
6040 sub get_current_qemu_machine {
6041     my ($vmid) = @_;
6042
6043     my $cmd = { execute => 'query-machines', arguments => {} };
6044     my $res = vm_qmp_command($vmid, $cmd);
6045
6046     my ($current, $default);
6047     foreach my $e (@$res) {
6048         $default = $e->{name} if $e->{'is-default'};
6049         $current = $e->{name} if $e->{'is-current'};
6050     }
6051
6052     # fallback to the default machine if current is not supported by qemu
6053     return $current || $default || 'pc';
6054 }
6055
6056 sub qemu_machine_feature_enabled {
6057     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6058
6059     my $current_major;
6060     my $current_minor;
6061
6062     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6063
6064         $current_major = $3;
6065         $current_minor = $4;
6066
6067     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6068
6069         $current_major = $1;
6070         $current_minor = $2;
6071     }
6072
6073     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6074
6075
6076 }
6077
6078 sub qemu_machine_pxe {
6079     my ($vmid, $conf, $machine) = @_;
6080
6081     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6082
6083     foreach my $opt (keys %$conf) {
6084         next if $opt !~ m/^net(\d+)$/;
6085         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6086         next if !$net;
6087         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6088         return $machine.".pxe" if $romfile =~ m/pxe/;
6089         last;
6090     }
6091
6092     return $machine;
6093 }
6094
6095 sub qemu_use_old_bios_files {
6096     my ($machine_type) = @_;
6097
6098     return if !$machine_type;
6099
6100     my $use_old_bios_files = undef;
6101
6102     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6103         $machine_type = $1;
6104         $use_old_bios_files = 1;
6105     } else {
6106         my $kvmver = kvm_user_version();
6107         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6108         # load new efi bios files on migration. So this hack is required to allow
6109         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6110         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6111         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6112     }
6113
6114     return ($use_old_bios_files, $machine_type);
6115 }
6116
6117 sub lspci {
6118
6119     my $devices = {};
6120
6121     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6122             my (undef, $id, $function) = @_;
6123             my $res = { id => $id, function => $function};
6124             push @{$devices->{$id}}, $res;
6125     });
6126
6127     return $devices;
6128 }
6129
6130 sub vm_iothreads_list {
6131     my ($vmid) = @_;
6132
6133     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6134
6135     my $iothreads = {};
6136     foreach my $iothread (@$res) {
6137         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6138     }
6139
6140     return $iothreads;
6141 }
6142
6143 sub scsihw_infos {
6144     my ($conf, $drive) = @_;
6145
6146     my $maxdev = 0;
6147
6148     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6149         $maxdev = 7;
6150     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6151         $maxdev = 1;
6152     } else {
6153         $maxdev = 256;
6154     }
6155
6156     my $controller = int($drive->{index} / $maxdev);
6157     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6158
6159     return ($maxdev, $controller, $controller_prefix);
6160 }
6161
6162 # bash completion helper
6163
6164 sub complete_backup_archives {
6165     my ($cmdname, $pname, $cvalue) = @_;
6166
6167     my $cfg = PVE::Storage::config();
6168
6169     my $storeid;
6170
6171     if ($cvalue =~ m/^([^:]+):/) {
6172         $storeid = $1;
6173     }
6174
6175     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6176
6177     my $res = [];
6178     foreach my $id (keys %$data) {
6179         foreach my $item (@{$data->{$id}}) {
6180             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6181             push @$res, $item->{volid} if defined($item->{volid});
6182         }
6183     }
6184
6185     return $res;
6186 }
6187
6188 my $complete_vmid_full = sub {
6189     my ($running) = @_;
6190
6191     my $idlist = vmstatus();
6192
6193     my $res = [];
6194
6195     foreach my $id (keys %$idlist) {
6196         my $d = $idlist->{$id};
6197         if (defined($running)) {
6198             next if $d->{template};
6199             next if $running && $d->{status} ne 'running';
6200             next if !$running && $d->{status} eq 'running';
6201         }
6202         push @$res, $id;
6203
6204     }
6205     return $res;
6206 };
6207
6208 sub complete_vmid {
6209     return &$complete_vmid_full();
6210 }
6211
6212 sub complete_vmid_stopped {
6213     return &$complete_vmid_full(0);
6214 }
6215
6216 sub complete_vmid_running {
6217     return &$complete_vmid_full(1);
6218 }
6219
6220 sub complete_storage {
6221
6222     my $cfg = PVE::Storage::config();
6223     my $ids = $cfg->{ids};
6224
6225     my $res = [];
6226     foreach my $sid (keys %$ids) {
6227         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6228         next if !$ids->{$sid}->{content}->{images};
6229         push @$res, $sid;
6230     }
6231
6232     return $res;
6233 }
6234
6235 1;