PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35 use URI::Escape;
  36
  37 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  38
  39 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  40
  41 # Note about locking: we use flock on the config file protect
  42 # against concurent actions.
  43 # Aditionaly, we have a 'lock' setting in the config file. This
  44 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  45 # allowed when such lock is set. But you can ignore this kind of
  46 # lock with the --skiplock flag.
  47
  48 cfs_register_file('/qemu-server/',
  49                   \&parse_vm_config,
  50                   \&write_vm_config);
  51
  52 PVE::JSONSchema::register_standard_option('skiplock', {
  53     description => "Ignore locks - only root is allowed to use this option.",
  54     type => 'boolean',
  55     optional => 1,
  56 });
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 #no warnings 'redefine';
  72
  73 sub cgroups_write {
  74    my ($controller, $vmid, $option, $value) = @_;
  75
  76    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  77    PVE::ProcFSTools::write_proc_entry($path, $value);
  78
  79 }
  80
  81 my $nodename = PVE::INotify::nodename();
  82
  83 mkdir "/etc/pve/nodes/$nodename";
  84 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  85 mkdir $confdir;
  86
  87 my $var_run_tmpdir = "/var/run/qemu-server";
  88 mkdir $var_run_tmpdir;
  89
  90 my $lock_dir = "/var/lock/qemu-server";
  91 mkdir $lock_dir;
  92
  93 my $pcisysfs = "/sys/bus/pci";
  94
  95 my $cpu_fmt = {
  96     cputype => {
  97         description => "Emulated CPU type.",
  98         type => 'string',
  99         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 100         format_description => 'cputype',
 101         default => 'kvm64',
 102         default_key => 1,
 103     },
 104     hidden => {
 105         description => "Do not identify as a KVM virtual machine.",
 106         type => 'boolean',
 107         optional => 1,
 108         default => 0
 109     },
 110 };
 111
 112 my $confdesc = {
 113     onboot => {
 114         optional => 1,
 115         type => 'boolean',
 116         description => "Specifies whether a VM will be started during system bootup.",
 117         default => 0,
 118     },
 119     autostart => {
 120         optional => 1,
 121         type => 'boolean',
 122         description => "Automatic restart after crash (currently ignored).",
 123         default => 0,
 124     },
 125     hotplug => {
 126         optional => 1,
 127         type => 'string', format => 'pve-hotplug-features',
 128         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 129         default => 'network,disk,usb',
 130     },
 131     reboot => {
 132         optional => 1,
 133         type => 'boolean',
 134         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 135         default => 1,
 136     },
 137     lock => {
 138         optional => 1,
 139         type => 'string',
 140         description => "Lock/unlock the VM.",
 141         enum => [qw(migrate backup snapshot rollback)],
 142     },
 143     cpulimit => {
 144         optional => 1,
 145         type => 'number',
 146         description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 147         minimum => 0,
 148         maximum => 128,
 149         default => 0,
 150     },
 151     cpuunits => {
 152         optional => 1,
 153         type => 'integer',
 154         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 155         minimum => 0,
 156         maximum => 500000,
 157         default => 1000,
 158     },
 159     memory => {
 160         optional => 1,
 161         type => 'integer',
 162         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 163         minimum => 16,
 164         default => 512,
 165     },
 166     balloon => {
 167         optional => 1,
 168         type => 'integer',
 169         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 170         minimum => 0,
 171     },
 172     shares => {
 173         optional => 1,
 174         type => 'integer',
 175         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 176         minimum => 0,
 177         maximum => 50000,
 178         default => 1000,
 179     },
 180     keyboard => {
 181         optional => 1,
 182         type => 'string',
 183         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 184         enum => PVE::Tools::kvmkeymaplist(),
 185         default => 'en-us',
 186     },
 187     name => {
 188         optional => 1,
 189         type => 'string', format => 'dns-name',
 190         description => "Set a name for the VM. Only used on the configuration web interface.",
 191     },
 192     scsihw => {
 193         optional => 1,
 194         type => 'string',
 195         description => "scsi controller model",
 196         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 197         default => 'lsi',
 198     },
 199     description => {
 200         optional => 1,
 201         type => 'string',
 202         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 203     },
 204     ostype => {
 205         optional => 1,
 206         type => 'string',
 207         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 208         description => <<EODESC,
 209 Used to enable special optimization/features for specific
 210 operating systems:
 211
 212 other  => unspecified OS
 213 wxp    => Microsoft Windows XP
 214 w2k    => Microsoft Windows 2000
 215 w2k3   => Microsoft Windows 2003
 216 w2k8   => Microsoft Windows 2008
 217 wvista => Microsoft Windows Vista
 218 win7   => Microsoft Windows 7
 219 win8   => Microsoft Windows 8/2012
 220 l24    => Linux 2.4 Kernel
 221 l26    => Linux 2.6/3.X Kernel
 222 solaris => solaris/opensolaris/openindiania kernel
 223
 224 other|l24|l26|solaris                       ... no special behaviour
 225 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 226 EODESC
 227     },
 228     boot => {
 229         optional => 1,
 230         type => 'string',
 231         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 232         pattern => '[acdn]{1,4}',
 233         default => 'cdn',
 234     },
 235     bootdisk => {
 236         optional => 1,
 237         type => 'string', format => 'pve-qm-bootdisk',
 238         description => "Enable booting from specified disk.",
 239         pattern => '(ide|sata|scsi|virtio)\d+',
 240     },
 241     smp => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "The number of CPUs. Please use option -sockets instead.",
 245         minimum => 1,
 246         default => 1,
 247     },
 248     sockets => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "The number of CPU sockets.",
 252         minimum => 1,
 253         default => 1,
 254     },
 255     cores => {
 256         optional => 1,
 257         type => 'integer',
 258         description => "The number of cores per socket.",
 259         minimum => 1,
 260         default => 1,
 261     },
 262     numa => {
 263         optional => 1,
 264         type => 'boolean',
 265         description => "Enable/disable NUMA.",
 266         default => 0,
 267     },
 268     vcpus => {
 269         optional => 1,
 270         type => 'integer',
 271         description => "Number of hotplugged vcpus.",
 272         minimum => 1,
 273         default => 0,
 274     },
 275     acpi => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Enable/disable ACPI.",
 279         default => 1,
 280     },
 281     agent => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Enable/disable Qemu GuestAgent.",
 285         default => 0,
 286     },
 287     kvm => {
 288         optional => 1,
 289         type => 'boolean',
 290         description => "Enable/disable KVM hardware virtualization.",
 291         default => 1,
 292     },
 293     tdf => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Enable/disable time drift fix.",
 297         default => 0,
 298     },
 299     localtime => {
 300         optional => 1,
 301         type => 'boolean',
 302         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 303     },
 304     freeze => {
 305         optional => 1,
 306         type => 'boolean',
 307         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 308     },
 309     vga => {
 310         optional => 1,
 311         type => 'string',
 312         description => "Select the VGA type. If you want to use high resolution" .
 313             " modes (>= 1280x1024x16) then you should use the options " .
 314             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 315             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 316             "display sever. For win* OS you can select how many independent " .
 317             "displays you want, Linux guests can add displays them self. " .
 318             "You can also run without any graphic card, using a serial device" .
 319             " as terminal.",
 320         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 321     },
 322     watchdog => {
 323         optional => 1,
 324         type => 'string', format => 'pve-qm-watchdog',
 325         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 326         description => "Create a virtual hardware watchdog device. Once enabled" .
 327             " (by a guest action), the watchdog must be periodically polled " .
 328             "by an agent inside the guest or else the watchdog will reset " .
 329             "the guest (or execute the respective action specified)",
 330     },
 331     startdate => {
 332         optional => 1,
 333         type => 'string',
 334         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 335         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 336         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 337         default => 'now',
 338     },
 339     startup =>  get_standard_option('pve-startup-order'),
 340     template => {
 341         optional => 1,
 342         type => 'boolean',
 343         description => "Enable/disable Template.",
 344         default => 0,
 345     },
 346     args => {
 347         optional => 1,
 348         type => 'string',
 349         description => <<EODESCR,
 350 NOTE: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 351
 352 args: -no-reboot -no-hpet
 353 EODESCR
 354     },
 355     tablet => {
 356         optional => 1,
 357         type => 'boolean',
 358         default => 1,
 359         description => "Enable/disable the USB tablet device. This device is " .
 360             "usually needed to allow absolute mouse positioning with VNC. " .
 361             "Else the mouse runs out of sync with normal VNC clients. " .
 362             "If you're running lots of console-only guests on one host, " .
 363             "you may consider disabling this to save some context switches. " .
 364             "This is turned off by default if you use spice (-vga=qxl).",
 365     },
 366     migrate_speed => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 370         minimum => 0,
 371         default => 0,
 372     },
 373     migrate_downtime => {
 374         optional => 1,
 375         type => 'number',
 376         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 377         minimum => 0,
 378         default => 0.1,
 379     },
 380     cdrom => {
 381         optional => 1,
 382         type => 'string', format => 'pve-qm-drive',
 383         typetext => 'volume',
 384         description => "This is an alias for option -ide2",
 385     },
 386     cpu => {
 387         optional => 1,
 388         description => "Emulated CPU type.",
 389         type => 'string',
 390         format => $cpu_fmt,
 391     },
 392     parent => get_standard_option('pve-snapshot-name', {
 393         optional => 1,
 394         description => "Parent snapshot name. This is used internally, and should not be modified.",
 395     }),
 396     snaptime => {
 397         optional => 1,
 398         description => "Timestamp for snapshots.",
 399         type => 'integer',
 400         minimum => 0,
 401     },
 402     vmstate => {
 403         optional => 1,
 404         type => 'string', format => 'pve-volume-id',
 405         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 406     },
 407     machine => {
 408         description => "Specific the Qemu machine type.",
 409         type => 'string',
 410         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 411         maxLength => 40,
 412         optional => 1,
 413     },
 414     smbios1 => {
 415         description => "Specify SMBIOS type 1 fields.",
 416         type => 'string', format => 'pve-qm-smbios1',
 417         maxLength => 256,
 418         optional => 1,
 419     },
 420     protection => {
 421         optional => 1,
 422         type => 'boolean',
 423         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 424         default => 0,
 425     },
 426     bios => {
 427         optional => 1,
 428         type => 'string',
 429         enum => [ qw(seabios ovmf) ],
 430         description => "Select BIOS implementation.",
 431         default => 'seabios',
 432     },
 433 };
 434
 435 # what about other qemu settings ?
 436 #cpu => 'string',
 437 #machine => 'string',
 438 #fda => 'file',
 439 #fdb => 'file',
 440 #mtdblock => 'file',
 441 #sd => 'file',
 442 #pflash => 'file',
 443 #snapshot => 'bool',
 444 #bootp => 'file',
 445 ##tftp => 'dir',
 446 ##smb => 'dir',
 447 #kernel => 'file',
 448 #append => 'string',
 449 #initrd => 'file',
 450 ##soundhw => 'string',
 451
 452 while (my ($k, $v) = each %$confdesc) {
 453     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 454 }
 455
 456 my $MAX_IDE_DISKS = 4;
 457 my $MAX_SCSI_DISKS = 14;
 458 my $MAX_VIRTIO_DISKS = 16;
 459 my $MAX_SATA_DISKS = 6;
 460 my $MAX_USB_DEVICES = 5;
 461 my $MAX_NETS = 32;
 462 my $MAX_UNUSED_DISKS = 8;
 463 my $MAX_HOSTPCI_DEVICES = 4;
 464 my $MAX_SERIAL_PORTS = 4;
 465 my $MAX_PARALLEL_PORTS = 3;
 466 my $MAX_NUMA = 8;
 467 my $MAX_MEM = 4194304;
 468 my $STATICMEM = 1024;
 469
 470 my $numa_fmt = {
 471     cpus => {
 472         type => "string",
 473         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 474         description => "CPUs accessing this numa node.",
 475         format_description => "id[-id];...",
 476     },
 477     memory => {
 478         type => "number",
 479         description => "Amount of memory this numa node provides.",
 480         format_description => "mb",
 481         optional => 1,
 482     },
 483     hostnodes => {
 484         type => "string",
 485         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 486         description => "host numa nodes to use",
 487         format_description => "id[-id];...",
 488         optional => 1,
 489     },
 490     policy => {
 491         type => 'string',
 492         enum => [qw(preferred bind interleave)],
 493         format_description => 'preferred|bind|interleave',
 494         description => "numa allocation policy.",
 495         optional => 1,
 496     },
 497 };
 498 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 499 my $numadesc = {
 500     optional => 1,
 501     type => 'string', format => $numa_fmt,
 502     description => "numa topology",
 503 };
 504 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 505
 506 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 507     $confdesc->{"numa$i"} = $numadesc;
 508 }
 509
 510 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 511                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 512                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 513 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 514
 515 my $net_fmt = {
 516     macaddr => {
 517         type => 'string',
 518         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 519         description => "MAC address",
 520         format_description => "XX:XX:XX:XX:XX:XX",
 521         optional => 1,
 522     },
 523     model => { alias => 'macaddr', default_key => 1 },
 524     (map { $_ => { group => 'model' } } @$nic_model_list),
 525     bridge => {
 526         type => 'string',
 527         description => 'Bridge to attach the network device to.',
 528         format_description => 'bridge',
 529         optional => 1,
 530     },
 531     queues => {
 532         type => 'integer',
 533         minimum => 0, maximum => 16,
 534         description => 'Number of packet queues to be used on the device.',
 535         format_description => 'number',
 536         optional => 1,
 537     },
 538     rate => {
 539         type => 'number',
 540         minimum => 0,
 541         description => 'Rate limit in mbps as floating point number.',
 542         format_description => 'mbps',
 543         optional => 1,
 544     },
 545     tag => {
 546         type => 'integer',
 547         minimum => 2, maximum => 4094,
 548         description => 'VLAN tag to apply to packets on this interface.',
 549         format_description => 'vlanid',
 550         optional => 1,
 551     },
 552     trunks => {
 553         type => 'string',
 554         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 555         description => 'VLAN trunks to pass through this interface.',
 556         format_description => 'id;id...',
 557         optional => 1,
 558     },
 559     firewall => {
 560         type => 'boolean',
 561         description => 'Whether this interface should be protected by the firewall.',
 562         format_description => '0|1',
 563         optional => 1,
 564     },
 565     link_down => {
 566         type => 'boolean',
 567         description => 'Whether this interface should be DISconnected (like pulling the plug).',
 568         format_description => '0|1',
 569         optional => 1,
 570     },
 571 };
 572 my $netdesc = {
 573     optional => 1,
 574     type => 'string', format => 'pve-qm-net',
 575     description => <<EODESCR,
 576 Specify network devices.
 577
 578 MODEL is one of: $nic_model_list_txt
 579
 580 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 581 automatically generated if not specified.
 582
 583 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 584
 585 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 586
 587 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 588
 589 10.0.2.2   Gateway
 590 10.0.2.3   DNS Server
 591 10.0.2.4   SMB Server
 592
 593 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 594
 595 EODESCR
 596 };
 597 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 598
 599 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 600     $confdesc->{"net$i"} = $netdesc;
 601 }
 602
 603 PVE::JSONSchema::register_format('pve-volume-id-or-none', \&verify_volume_id_or_none);
 604 sub verify_volume_id_or_none {
 605     my ($volid, $noerr) = @_;
 606
 607     return $volid if $volid eq 'none';
 608     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 609     if ($@) {
 610         return undef if $noerr;
 611         die $@;
 612     }
 613     return $volid;
 614 }
 615
 616 my $drivename_hash;
 617
 618 my %drivedesc_base = (
 619     volume => { alias => 'file' },
 620     file => {
 621         type => 'string',
 622         format => 'pve-volume-id-or-none',
 623         default_key => 1,
 624         format_description => 'volume',
 625         description => "The drive's backing volume.",
 626     },
 627     media => {
 628         type => 'string',
 629         format_description => 'cdrom|disk',
 630         enum => [qw(cdrom disk)],
 631         description => "The drive's media type.",
 632         default => 'disk',
 633         optional => 1
 634     },
 635     cyls => {
 636         type => 'integer',
 637         format_description => 'count',
 638         description => "Force the drive's physical geometry to have a specific cylinder count.",
 639         optional => 1
 640     },
 641     heads => {
 642         type => 'integer',
 643         format_description => 'count',
 644         description => "Force the drive's physical geometry to have a specific head count.",
 645         optional => 1
 646     },
 647     secs => {
 648         type => 'integer',
 649         format_description => 'count',
 650         description => "Force the drive's physical geometry to have a specific sector count.",
 651         optional => 1
 652     },
 653     trans => {
 654         type => 'string',
 655         format_description => 'none|lba|auto',
 656         enum => [qw(none lba auto)],
 657         description => "Force disk geometry bios translation mode.",
 658         optional => 1,
 659     },
 660     snapshot => {
 661         type => 'boolean',
 662         format_description => 'on|off',
 663         description => "Whether the drive should be included when making snapshots.",
 664         optional => 1,
 665     },
 666     cache => {
 667         type => 'string',
 668         format_description => 'none|writethrough|writeback|unsafe|directsync',
 669         enum => [qw(none writethrough writeback unsafe directsync)],
 670         description => "The drive's cache mode",
 671         optional => 1,
 672     },
 673     format => {
 674         type => 'string',
 675         format_description => 'drive format',
 676         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 677         description => "The drive's backing file's data format.",
 678         optional => 1,
 679     },
 680     size => {
 681         type => 'string',
 682         format => 'disk-size',
 683         description => "Disk size. This is purely informational and has no effect.",
 684         optional => 1,
 685     },
 686     backup => {
 687         type => 'boolean',
 688         format_description => 'on|off',
 689         description => "Whether the drive should be included when making backups.",
 690         optional => 1,
 691     },
 692     werror => {
 693         type => 'string',
 694         format_description => 'enospc|ignore|report|stop',
 695         enum => [qw(enospc ignore report stop)],
 696         description => 'Write error action.',
 697         optional => 1,
 698     },
 699     aio => {
 700         type => 'string',
 701         format_description => 'native|threads',
 702         enum => [qw(native threads)],
 703         description => 'AIO type to use.',
 704         optional => 1,
 705     },
 706     discard => {
 707         type => 'string',
 708         format_description => 'ignore|on',
 709         enum => [qw(ignore on)],
 710         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 711         optional => 1,
 712     },
 713     detect_zeroes => {
 714         type => 'boolean',
 715         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 716         optional => 1,
 717     },
 718     serial => {
 719         type => 'string',
 720         format => 'urlencoded',
 721         format_description => 'serial',
 722         maxLength => 20*3, # *3 since it's %xx url enoded
 723         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 724         optional => 1,
 725     }
 726 );
 727
 728 my %rerror_fmt = (
 729     rerror => {
 730         type => 'string',
 731         format_description => 'ignore|report|stop',
 732         enum => [qw(ignore report stop)],
 733         description => 'Read error action.',
 734         optional => 1,
 735     },
 736 );
 737
 738 my %iothread_fmt = ( iothread => {
 739         type => 'boolean',
 740         format_description => 'off|on',
 741         description => "Whether to use iothreads for this drive",
 742         optional => 1,
 743 });
 744
 745 my %model_fmt = (
 746     model => {
 747         type => 'string',
 748         format => 'urlencoded',
 749         format_description => 'model',
 750         maxLength => 40*3, # *3 since it's %xx url enoded
 751         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 752         optional => 1,
 753     },
 754 );
 755
 756 my %queues_fmt = (
 757     queues => {
 758         type => 'integer',
 759         format_description => 'nbqueues',
 760         description => "Number of queues.",
 761         minimum => 2,
 762         optional => 1
 763     }
 764 );
 765
 766 my $add_throttle_desc = sub {
 767     my ($key, $type, $what, $size, $longsize) = @_;
 768     $drivedesc_base{$key} = {
 769         type => $type,
 770         format_description => $size,
 771         description => "Maximum $what speed in $longsize per second.",
 772         optional => 1,
 773     };
 774 };
 775 # throughput: (leaky bucket)
 776 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 777 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 778 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 779 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 780 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 781 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 782 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 783 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 784 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 785
 786 # pools: (pool of IO before throttling starts taking effect)
 787 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 788 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 789 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 790 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 791 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 792 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 793
 794 my $ide_fmt = {
 795     %drivedesc_base,
 796     %rerror_fmt,
 797     %model_fmt,
 798 };
 799
 800 my $idedesc = {
 801     optional => 1,
 802     type => 'string', format => $ide_fmt,
 803     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 804 };
 805 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 806
 807 my $scsi_fmt = {
 808     %drivedesc_base,
 809     %iothread_fmt,
 810     %queues_fmt,
 811 };
 812 my $scsidesc = {
 813     optional => 1,
 814     type => 'string', format => $scsi_fmt,
 815     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 816 };
 817 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 818
 819 my $sata_fmt = {
 820     %drivedesc_base,
 821     %rerror_fmt,
 822 };
 823 my $satadesc = {
 824     optional => 1,
 825     type => 'string', format => $sata_fmt,
 826     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 827 };
 828 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 829
 830 my $virtio_fmt = {
 831     %drivedesc_base,
 832     %iothread_fmt,
 833     %rerror_fmt,
 834 };
 835 my $virtiodesc = {
 836     optional => 1,
 837     type => 'string', format => $virtio_fmt,
 838     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 839 };
 840 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 841
 842 my $alldrive_fmt = {
 843     %drivedesc_base,
 844     %rerror_fmt,
 845     %iothread_fmt,
 846     %model_fmt,
 847     %queues_fmt,
 848 };
 849
 850 my $usb_fmt = {
 851     host => {
 852         default_key => 1,
 853         type => 'string', format => 'pve-qm-usb-device',
 854         format_description => 'HOSTUSBDEVICE|spice',
 855         description => 'The Host USB device or port or the value spice',
 856     },
 857     usb3 => {
 858         optional => 1,
 859         type => 'boolean',
 860         format_description => 'yes|no',
 861         description => 'Specifies whether if given host option is a USB3 device or port',
 862     },
 863 };
 864
 865 my $usbdesc = {
 866     optional => 1,
 867     type => 'string', format => $usb_fmt,
 868     description => <<EODESCR,
 869 Configure an USB device (n is 0 to 4). This can be used to
 870 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 871
 872 'bus-port(.port)*' (decimal numbers) or
 873 'vendor_id:product_id' (hexadeciaml numbers) or
 874 'spice'
 875
 876 You can use the 'lsusb -t' command to list existing usb devices.
 877
 878 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 879
 880 The value 'spice' can be used to add a usb redirection devices for spice.
 881
 882 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 883
 884 EODESCR
 885 };
 886 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 887
 888 my $hostpcidesc = {
 889         optional => 1,
 890         type => 'string', format => 'pve-qm-hostpci',
 891         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 892         description => <<EODESCR,
 893 Map host pci devices. HOSTPCIDEVICE syntax is:
 894
 895 'bus:dev.func' (hexadecimal numbers)
 896
 897 You can us the 'lspci' command to list existing pci devices.
 898
 899 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 900
 901 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 902
 903 Experimental: user reported problems with this option.
 904 EODESCR
 905 };
 906 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 907
 908 my $serialdesc = {
 909         optional => 1,
 910         type => 'string',
 911         pattern => '(/dev/.+|socket)',
 912         description =>  <<EODESCR,
 913 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 914
 915 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 916
 917 Experimental: user reported problems with this option.
 918 EODESCR
 919 };
 920
 921 my $paralleldesc= {
 922         optional => 1,
 923         type => 'string',
 924         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 925         description =>  <<EODESCR,
 926 Map host parallel devices (n is 0 to 2).
 927
 928 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 929
 930 Experimental: user reported problems with this option.
 931 EODESCR
 932 };
 933
 934 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 935     $confdesc->{"parallel$i"} = $paralleldesc;
 936 }
 937
 938 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 939     $confdesc->{"serial$i"} = $serialdesc;
 940 }
 941
 942 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 943     $confdesc->{"hostpci$i"} = $hostpcidesc;
 944 }
 945
 946 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 947     $drivename_hash->{"ide$i"} = 1;
 948     $confdesc->{"ide$i"} = $idedesc;
 949 }
 950
 951 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 952     $drivename_hash->{"sata$i"} = 1;
 953     $confdesc->{"sata$i"} = $satadesc;
 954 }
 955
 956 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 957     $drivename_hash->{"scsi$i"} = 1;
 958     $confdesc->{"scsi$i"} = $scsidesc ;
 959 }
 960
 961 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 962     $drivename_hash->{"virtio$i"} = 1;
 963     $confdesc->{"virtio$i"} = $virtiodesc;
 964 }
 965
 966 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 967     $confdesc->{"usb$i"} = $usbdesc;
 968 }
 969
 970 my $unuseddesc = {
 971     optional => 1,
 972     type => 'string', format => 'pve-volume-id',
 973     description => "Reference to unused volumes.",
 974 };
 975
 976 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 977     $confdesc->{"unused$i"} = $unuseddesc;
 978 }
 979
 980 my $kvm_api_version = 0;
 981
 982 sub kvm_version {
 983
 984     return $kvm_api_version if $kvm_api_version;
 985
 986     my $fh = IO::File->new("</dev/kvm") ||
 987         return 0;
 988
 989     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 990         $kvm_api_version = $v;
 991     }
 992
 993     $fh->close();
 994
 995     return  $kvm_api_version;
 996 }
 997
 998 my $kvm_user_version;
 999
1000 sub kvm_user_version {
1001
1002     return $kvm_user_version if $kvm_user_version;
1003
1004     $kvm_user_version = 'unknown';
1005
1006     my $code = sub {
1007         my $line = shift;
1008         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1009             $kvm_user_version = $2;
1010         }
1011     };
1012
1013     eval { run_command("kvm -version", outfunc => $code); };
1014     warn $@ if $@;
1015
1016     return $kvm_user_version;
1017
1018 }
1019
1020 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1021
1022 sub valid_drive_names {
1023     # order is important - used to autoselect boot disk
1024     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1025             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1026             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1027             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1028 }
1029
1030 sub is_valid_drivename {
1031     my $dev = shift;
1032
1033     return defined($drivename_hash->{$dev});
1034 }
1035
1036 sub option_exists {
1037     my $key = shift;
1038     return defined($confdesc->{$key});
1039 }
1040
1041 sub nic_models {
1042     return $nic_model_list;
1043 }
1044
1045 sub os_list_description {
1046
1047     return {
1048         other => 'Other',
1049         wxp => 'Windows XP',
1050         w2k => 'Windows 2000',
1051         w2k3 =>, 'Windows 2003',
1052         w2k8 => 'Windows 2008',
1053         wvista => 'Windows Vista',
1054         win7 => 'Windows 7',
1055         win8 => 'Windows 8/2012',
1056         l24 => 'Linux 2.4',
1057         l26 => 'Linux 2.6',
1058     };
1059 }
1060
1061 my $cdrom_path;
1062
1063 sub get_cdrom_path {
1064
1065     return  $cdrom_path if $cdrom_path;
1066
1067     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1068     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1069     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1070 }
1071
1072 sub get_iso_path {
1073     my ($storecfg, $vmid, $cdrom) = @_;
1074
1075     if ($cdrom eq 'cdrom') {
1076         return get_cdrom_path();
1077     } elsif ($cdrom eq 'none') {
1078         return '';
1079     } elsif ($cdrom =~ m|^/|) {
1080         return $cdrom;
1081     } else {
1082         return PVE::Storage::path($storecfg, $cdrom);
1083     }
1084 }
1085
1086 # try to convert old style file names to volume IDs
1087 sub filename_to_volume_id {
1088     my ($vmid, $file, $media) = @_;
1089
1090     if (!($file eq 'none' || $file eq 'cdrom' ||
1091           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1092
1093         return undef if $file =~ m|/|;
1094
1095         if ($media && $media eq 'cdrom') {
1096             $file = "local:iso/$file";
1097         } else {
1098             $file = "local:$vmid/$file";
1099         }
1100     }
1101
1102     return $file;
1103 }
1104
1105 sub verify_media_type {
1106     my ($opt, $vtype, $media) = @_;
1107
1108     return if !$media;
1109
1110     my $etype;
1111     if ($media eq 'disk') {
1112         $etype = 'images';
1113     } elsif ($media eq 'cdrom') {
1114         $etype = 'iso';
1115     } else {
1116         die "internal error";
1117     }
1118
1119     return if ($vtype eq $etype);
1120
1121     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1122 }
1123
1124 sub cleanup_drive_path {
1125     my ($opt, $storecfg, $drive) = @_;
1126
1127     # try to convert filesystem paths to volume IDs
1128
1129     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1130         ($drive->{file} !~ m|^/dev/.+|) &&
1131         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1132         ($drive->{file} !~ m/^\d+$/)) {
1133         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1134         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1135         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1136         verify_media_type($opt, $vtype, $drive->{media});
1137         $drive->{file} = $volid;
1138     }
1139
1140     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1141 }
1142
1143 sub parse_hotplug_features {
1144     my ($data) = @_;
1145
1146     my $res = {};
1147
1148     return $res if $data eq '0';
1149
1150     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1151
1152     foreach my $feature (PVE::Tools::split_list($data)) {
1153         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1154             $res->{$1} = 1;
1155         } else {
1156             warn "ignoring unknown hotplug feature '$feature'\n";
1157         }
1158     }
1159     return $res;
1160 }
1161
1162 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1163 sub pve_verify_hotplug_features {
1164     my ($value, $noerr) = @_;
1165
1166     return $value if parse_hotplug_features($value);
1167
1168     return undef if $noerr;
1169
1170     die "unable to parse hotplug option\n";
1171 }
1172
1173 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1174 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1175 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1176 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1177 #        [,iothread=on][,serial=serial][,model=model]
1178
1179 sub parse_drive {
1180     my ($key, $data) = @_;
1181
1182     my ($interface, $index);
1183
1184     if ($key =~ m/^([^\d]+)(\d+)$/) {
1185         $interface = $1;
1186         $index = $2;
1187     } else {
1188         return undef;
1189     }
1190
1191     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1192                                      : $confdesc->{$key}->{format};
1193     if (!$desc) {
1194         warn "invalid drive key: $key\n";
1195         return undef;
1196     }
1197     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1198     return undef if !$res;
1199     $res->{interface} = $interface;
1200     $res->{index} = $index;
1201
1202     my $error = 0;
1203     foreach my $opt (qw(bps bps_rd bps_wr)) {
1204         if (my $bps = defined(delete $res->{$opt})) {
1205             if (defined($res->{"m$opt"})) {
1206                 warn "both $opt and m$opt specified\n";
1207                 ++$error;
1208                 next;
1209             }
1210             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1211         }
1212     }
1213     return undef if $error;
1214
1215     return undef if $res->{mbps_rd} && $res->{mbps};
1216     return undef if $res->{mbps_wr} && $res->{mbps};
1217     return undef if $res->{iops_rd} && $res->{iops};
1218     return undef if $res->{iops_wr} && $res->{iops};
1219
1220     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1221         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1222         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1223         return undef if $res->{interface} eq 'virtio';
1224     }
1225
1226     if (my $size = $res->{size}) {
1227         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1228     }
1229
1230     return $res;
1231 }
1232
1233 sub print_drive {
1234     my ($vmid, $drive) = @_;
1235     my $data = { %$drive };
1236     delete $data->{$_} for qw(index interface);
1237     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1238 }
1239
1240 sub scsi_inquiry {
1241     my($fh, $noerr) = @_;
1242
1243     my $SG_IO = 0x2285;
1244     my $SG_GET_VERSION_NUM = 0x2282;
1245
1246     my $versionbuf = "\x00" x 8;
1247     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1248     if (!$ret) {
1249         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1250         return undef;
1251     }
1252     my $version = unpack("I", $versionbuf);
1253     if ($version < 30000) {
1254         die "scsi generic interface too old\n"  if !$noerr;
1255         return undef;
1256     }
1257
1258     my $buf = "\x00" x 36;
1259     my $sensebuf = "\x00" x 8;
1260     my $cmd = pack("C x3 C x1", 0x12, 36);
1261
1262     # see /usr/include/scsi/sg.h
1263     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1264
1265     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1266                       length($sensebuf), 0, length($buf), $buf,
1267                       $cmd, $sensebuf, 6000);
1268
1269     $ret = ioctl($fh, $SG_IO, $packet);
1270     if (!$ret) {
1271         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1272         return undef;
1273     }
1274
1275     my @res = unpack($sg_io_hdr_t, $packet);
1276     if ($res[17] || $res[18]) {
1277         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1278         return undef;
1279     }
1280
1281     my $res = {};
1282     (my $byte0, my $byte1, $res->{vendor},
1283      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1284
1285     $res->{removable} = $byte1 & 128 ? 1 : 0;
1286     $res->{type} = $byte0 & 31;
1287
1288     return $res;
1289 }
1290
1291 sub path_is_scsi {
1292     my ($path) = @_;
1293
1294     my $fh = IO::File->new("+<$path") || return undef;
1295     my $res = scsi_inquiry($fh, 1);
1296     close($fh);
1297
1298     return $res;
1299 }
1300
1301 sub machine_type_is_q35 {
1302     my ($conf) = @_;
1303
1304     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1305 }
1306
1307 sub print_tabletdevice_full {
1308     my ($conf) = @_;
1309
1310     my $q35 = machine_type_is_q35($conf);
1311
1312     # we use uhci for old VMs because tablet driver was buggy in older qemu
1313     my $usbbus = $q35 ? "ehci" : "uhci";
1314
1315     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1316 }
1317
1318 sub print_drivedevice_full {
1319     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1320
1321     my $device = '';
1322     my $maxdev = 0;
1323
1324     if ($drive->{interface} eq 'virtio') {
1325         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1326         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1327         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1328     } elsif ($drive->{interface} eq 'scsi') {
1329
1330         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1331         my $unit = $drive->{index} % $maxdev;
1332         my $devicetype = 'hd';
1333         my $path = '';
1334         if (drive_is_cdrom($drive)) {
1335             $devicetype = 'cd';
1336         } else {
1337             if ($drive->{file} =~ m|^/|) {
1338                 $path = $drive->{file};
1339                 if (my $info = path_is_scsi($path)) {
1340                     if ($info->{type} == 0) {
1341                         $devicetype = 'block';
1342                     } elsif ($info->{type} == 1) { # tape
1343                         $devicetype = 'generic';
1344                     }
1345                 }
1346             } else {
1347                  $path = PVE::Storage::path($storecfg, $drive->{file});
1348             }
1349
1350             if($path =~ m/^iscsi\:\/\//){
1351                 $devicetype = 'generic';
1352             }
1353         }
1354
1355         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1356             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1357         } else {
1358             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1359         }
1360
1361     } elsif ($drive->{interface} eq 'ide'){
1362         $maxdev = 2;
1363         my $controller = int($drive->{index} / $maxdev);
1364         my $unit = $drive->{index} % $maxdev;
1365         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1366
1367         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1368         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1369             $model = URI::Escape::uri_unescape($model);
1370             $device .= ",model=$model";
1371         }
1372     } elsif ($drive->{interface} eq 'sata'){
1373         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1374         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1375         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1376     } elsif ($drive->{interface} eq 'usb') {
1377         die "implement me";
1378         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1379     } else {
1380         die "unsupported interface type";
1381     }
1382
1383     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1384
1385     return $device;
1386 }
1387
1388 sub get_initiator_name {
1389     my $initiator;
1390
1391     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1392     while (defined(my $line = <$fh>)) {
1393         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1394         $initiator = $1;
1395         last;
1396     }
1397     $fh->close();
1398
1399     return $initiator;
1400 }
1401
1402 sub print_drive_full {
1403     my ($storecfg, $vmid, $drive) = @_;
1404
1405     my $path;
1406     my $volid = $drive->{file};
1407     my $format;
1408
1409     if (drive_is_cdrom($drive)) {
1410         $path = get_iso_path($storecfg, $vmid, $volid);
1411     } else {
1412         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1413         if ($storeid) {
1414             $path = PVE::Storage::path($storecfg, $volid);
1415             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1416             $format = qemu_img_format($scfg, $volname);
1417         } else {
1418             $path = $volid;
1419             $format = "raw";
1420         }
1421    }
1422
1423     my $opts = '';
1424     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1425     foreach my $o (@qemu_drive_options) {
1426         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1427     }
1428     if (my $serial = $drive->{serial}) {
1429         $serial = URI::Escape::uri_unescape($serial);
1430         $opts .= ",serial=$serial";
1431     }
1432
1433     $opts .= ",format=$format" if $format && !$drive->{format};
1434
1435     foreach my $o (qw(bps bps_rd bps_wr)) {
1436         my $v = $drive->{"m$o"};
1437         $opts .= ",$o=" . int($v*1024*1024) if $v;
1438     }
1439
1440     my $cache_direct = 0;
1441
1442     if (my $cache = $drive->{cache}) {
1443         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1444     } elsif (!drive_is_cdrom($drive)) {
1445         $opts .= ",cache=none";
1446         $cache_direct = 1;
1447     }
1448
1449     # aio native works only with O_DIRECT
1450     if (!$drive->{aio}) {
1451         if($cache_direct) {
1452             $opts .= ",aio=native";
1453         } else {
1454             $opts .= ",aio=threads";
1455         }
1456     }
1457
1458     if (!drive_is_cdrom($drive)) {
1459         my $detectzeroes;
1460         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1461             $detectzeroes = 'off';
1462         } elsif ($drive->{discard}) {
1463             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1464         } else {
1465             # This used to be our default with discard not being specified:
1466             $detectzeroes = 'on';
1467         }
1468         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1469     }
1470
1471     my $pathinfo = $path ? "file=$path," : '';
1472
1473     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1474 }
1475
1476 sub print_netdevice_full {
1477     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1478
1479     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1480
1481     my $device = $net->{model};
1482     if ($net->{model} eq 'virtio') {
1483          $device = 'virtio-net-pci';
1484      };
1485
1486     my $pciaddr = print_pci_addr("$netid", $bridges);
1487     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1488     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1489         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1490         my $vectors = $net->{queues} * 2 + 2;
1491         $tmpstr .= ",vectors=$vectors,mq=on";
1492     }
1493     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1494
1495     if ($use_old_bios_files) {
1496         my $romfile;
1497         if ($device eq 'virtio-net-pci') {
1498             $romfile = 'pxe-virtio.rom';
1499         } elsif ($device eq 'e1000') {
1500             $romfile = 'pxe-e1000.rom';
1501         } elsif ($device eq 'ne2k') {
1502             $romfile = 'pxe-ne2k_pci.rom';
1503         } elsif ($device eq 'pcnet') {
1504             $romfile = 'pxe-pcnet.rom';
1505         } elsif ($device eq 'rtl8139') {
1506             $romfile = 'pxe-rtl8139.rom';
1507         }
1508         $tmpstr .= ",romfile=$romfile" if $romfile;
1509     }
1510
1511     return $tmpstr;
1512 }
1513
1514 sub print_netdev_full {
1515     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1516
1517     my $i = '';
1518     if ($netid =~ m/^net(\d+)$/) {
1519         $i = int($1);
1520     }
1521
1522     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1523
1524     my $ifname = "tap${vmid}i$i";
1525
1526     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1527     die "interface name '$ifname' is too long (max 15 character)\n"
1528         if length($ifname) >= 16;
1529
1530     my $vhostparam = '';
1531     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1532
1533     my $vmname = $conf->{name} || "vm$vmid";
1534
1535     my $netdev = "";
1536     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1537
1538     if ($net->{bridge}) {
1539         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1540     } else {
1541         $netdev = "type=user,id=$netid,hostname=$vmname";
1542     }
1543
1544     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1545
1546     return $netdev;
1547 }
1548
1549 sub drive_is_cdrom {
1550     my ($drive) = @_;
1551
1552     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1553
1554 }
1555
1556 sub parse_number_sets {
1557     my ($set) = @_;
1558     my $res = [];
1559     foreach my $part (split(/;/, $set)) {
1560         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1561             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1562             push @$res, [ $1, $2 ];
1563         } else {
1564             die "invalid range: $part\n";
1565         }
1566     }
1567     return $res;
1568 }
1569
1570 sub parse_numa {
1571     my ($data) = @_;
1572
1573     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1574     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1575     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1576     return $res;
1577 }
1578
1579 sub parse_hostpci {
1580     my ($value) = @_;
1581
1582     return undef if !$value;
1583
1584
1585     my @list = split(/,/, $value);
1586     my $found;
1587
1588     my $res = {};
1589     foreach my $kv (@list) {
1590
1591         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1592             $found = 1;
1593             if(defined($4)){
1594                 push @{$res->{pciid}}, { id => $2 , function => $4};
1595
1596             }else{
1597                 my $pcidevices = lspci($2);
1598                 $res->{pciid} = $pcidevices->{$2};
1599             }
1600         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1601             $res->{rombar} = $1;
1602         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1603             $res->{'x-vga'} = $1;
1604         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1605             $res->{pcie} = 1 if $1 == 1;
1606         } else {
1607             warn "unknown hostpci setting '$kv'\n";
1608         }
1609     }
1610
1611     return undef if !$found;
1612
1613     return $res;
1614 }
1615
1616 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1617 sub parse_net {
1618     my ($data) = @_;
1619
1620     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1621     if ($@) {
1622         warn $@;
1623         return undef;
1624     }
1625     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1626     return $res;
1627 }
1628
1629 sub print_net {
1630     my $net = shift;
1631
1632     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1633 }
1634
1635 sub add_random_macs {
1636     my ($settings) = @_;
1637
1638     foreach my $opt (keys %$settings) {
1639         next if $opt !~ m/^net(\d+)$/;
1640         my $net = parse_net($settings->{$opt});
1641         next if !$net;
1642         $settings->{$opt} = print_net($net);
1643     }
1644 }
1645
1646 sub vm_is_volid_owner {
1647     my ($storecfg, $vmid, $volid) = @_;
1648
1649     if ($volid !~  m|^/|) {
1650         my ($path, $owner);
1651         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1652         if ($owner && ($owner == $vmid)) {
1653             return 1;
1654         }
1655     }
1656
1657     return undef;
1658 }
1659
1660 sub split_flagged_list {
1661     my $text = shift || '';
1662     $text =~ s/[,;]/ /g;
1663     $text =~ s/^\s+//;
1664     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1665 }
1666
1667 sub join_flagged_list {
1668     my ($how, $lst) = @_;
1669     join $how, map { $lst->{$_} . $_ } keys %$lst;
1670 }
1671
1672 sub vmconfig_delete_pending_option {
1673     my ($conf, $key, $force) = @_;
1674
1675     delete $conf->{pending}->{$key};
1676     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1677     $pending_delete_hash->{$key} = $force ? '!' : '';
1678     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1679 }
1680
1681 sub vmconfig_undelete_pending_option {
1682     my ($conf, $key) = @_;
1683
1684     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1685     delete $pending_delete_hash->{$key};
1686
1687     if (%$pending_delete_hash) {
1688         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1689     } else {
1690         delete $conf->{pending}->{delete};
1691     }
1692 }
1693
1694 sub vmconfig_register_unused_drive {
1695     my ($storecfg, $vmid, $conf, $drive) = @_;
1696
1697     if (!drive_is_cdrom($drive)) {
1698         my $volid = $drive->{file};
1699         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1700             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1701         }
1702     }
1703 }
1704
1705 sub vmconfig_cleanup_pending {
1706     my ($conf) = @_;
1707
1708     # remove pending changes when nothing changed
1709     my $changes;
1710     foreach my $opt (keys %{$conf->{pending}}) {
1711         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1712             $changes = 1;
1713             delete $conf->{pending}->{$opt};
1714         }
1715     }
1716
1717     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1718     my $pending_delete_hash = {};
1719     while (my ($opt, $force) = each %$current_delete_hash) {
1720         if (defined($conf->{$opt})) {
1721             $pending_delete_hash->{$opt} = $force;
1722         } else {
1723             $changes = 1;
1724         }
1725     }
1726
1727     if (%$pending_delete_hash) {
1728         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1729     } else {
1730         delete $conf->{pending}->{delete};
1731     }
1732
1733     return $changes;
1734 }
1735
1736 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1737 my $smbios1_fmt = {
1738     uuid => {
1739         type => 'string',
1740         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1741         format_description => 'UUID',
1742         optional => 1,
1743     },
1744     version => {
1745         type => 'string',
1746         pattern => '\S+',
1747         format_description => 'str',
1748         optional => 1,
1749     },
1750     serial => {
1751         type => 'string',
1752         pattern => '\S+',
1753         format_description => 'str',
1754         optional => 1,
1755     },
1756     manufacturer => {
1757         type => 'string',
1758         pattern => '\S+',
1759         format_description => 'name',
1760         optional => 1,
1761     },
1762     product => {
1763         type => 'string',
1764         pattern => '\S+',
1765         format_description => 'name',
1766         optional => 1,
1767     },
1768     sku => {
1769         type => 'string',
1770         pattern => '\S+',
1771         format_description => 'str',
1772         optional => 1,
1773     },
1774     family => {
1775         type => 'string',
1776         pattern => '\S+',
1777         format_description => 'str',
1778         optional => 1,
1779     },
1780 };
1781
1782 sub parse_smbios1 {
1783     my ($data) = @_;
1784
1785     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1786     warn $@ if $@;
1787     return $res;
1788 }
1789
1790 sub print_smbios1 {
1791     my ($smbios1) = @_;
1792     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1793 }
1794
1795 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1796
1797 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1798 sub verify_bootdisk {
1799     my ($value, $noerr) = @_;
1800
1801     return $value if is_valid_drivename($value);
1802
1803     return undef if $noerr;
1804
1805     die "invalid boot disk '$value'\n";
1806 }
1807
1808 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1809 sub verify_net {
1810     my ($value, $noerr) = @_;
1811
1812     return $value if parse_net($value);
1813
1814     return undef if $noerr;
1815
1816     die "unable to parse network options\n";
1817 }
1818
1819 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1820 sub verify_hostpci {
1821     my ($value, $noerr) = @_;
1822
1823     return $value if parse_hostpci($value);
1824
1825     return undef if $noerr;
1826
1827     die "unable to parse pci id\n";
1828 }
1829
1830 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1831 sub verify_watchdog {
1832     my ($value, $noerr) = @_;
1833
1834     return $value if parse_watchdog($value);
1835
1836     return undef if $noerr;
1837
1838     die "unable to parse watchdog options\n";
1839 }
1840
1841 sub parse_watchdog {
1842     my ($value) = @_;
1843
1844     return undef if !$value;
1845
1846     my $res = {};
1847
1848     foreach my $p (split(/,/, $value)) {
1849         next if $p =~ m/^\s*$/;
1850
1851         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1852             $res->{model} = $2;
1853         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1854             $res->{action} = $2;
1855         } else {
1856             return undef;
1857         }
1858     }
1859
1860     return $res;
1861 }
1862
1863 sub parse_usb_device {
1864     my ($value) = @_;
1865
1866     return undef if !$value;
1867
1868     my $res = {};
1869     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1870         $res->{vendorid} = $2;
1871         $res->{productid} = $4;
1872     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1873         $res->{hostbus} = $1;
1874         $res->{hostport} = $2;
1875     } elsif ($value =~ m/^spice$/i) {
1876         $res->{spice} = 1;
1877     } else {
1878         return undef;
1879     }
1880
1881     return $res;
1882 }
1883
1884 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1885 sub verify_usb_device {
1886     my ($value, $noerr) = @_;
1887
1888     return $value if parse_usb_device($value);
1889
1890     return undef if $noerr;
1891
1892     die "unable to parse usb device\n";
1893 }
1894
1895 # add JSON properties for create and set function
1896 sub json_config_properties {
1897     my $prop = shift;
1898
1899     foreach my $opt (keys %$confdesc) {
1900         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1901         $prop->{$opt} = $confdesc->{$opt};
1902     }
1903
1904     return $prop;
1905 }
1906
1907 sub check_type {
1908     my ($key, $value) = @_;
1909
1910     die "unknown setting '$key'\n" if !$confdesc->{$key};
1911
1912     my $type = $confdesc->{$key}->{type};
1913
1914     if (!defined($value)) {
1915         die "got undefined value\n";
1916     }
1917
1918     if ($value =~ m/[\n\r]/) {
1919         die "property contains a line feed\n";
1920     }
1921
1922     if ($type eq 'boolean') {
1923         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1924         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1925         die "type check ('boolean') failed - got '$value'\n";
1926     } elsif ($type eq 'integer') {
1927         return int($1) if $value =~ m/^(\d+)$/;
1928         die "type check ('integer') failed - got '$value'\n";
1929     } elsif ($type eq 'number') {
1930         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1931         die "type check ('number') failed - got '$value'\n";
1932     } elsif ($type eq 'string') {
1933         if (my $fmt = $confdesc->{$key}->{format}) {
1934             if ($fmt eq 'pve-qm-drive') {
1935                 # special case - we need to pass $key to parse_drive()
1936                 my $drive = parse_drive($key, $value);
1937                 return $value if $drive;
1938                 die "unable to parse drive options\n";
1939             }
1940             PVE::JSONSchema::check_format($fmt, $value);
1941             return $value;
1942         }
1943         $value =~ s/^\"(.*)\"$/$1/;
1944         return $value;
1945     } else {
1946         die "internal error"
1947     }
1948 }
1949
1950 sub check_iommu_support{
1951     #fixme : need to check IOMMU support
1952     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1953
1954     my $iommu=1;
1955     return $iommu;
1956
1957 }
1958
1959 sub touch_config {
1960     my ($vmid) = @_;
1961
1962     my $conf = PVE::QemuConfig->config_file($vmid);
1963     utime undef, undef, $conf;
1964 }
1965
1966 sub destroy_vm {
1967     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1968
1969     my $conffile = PVE::QemuConfig->config_file($vmid);
1970
1971     my $conf = PVE::QemuConfig->load_config($vmid);
1972
1973     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1974
1975     # only remove disks owned by this VM
1976     foreach_drive($conf, sub {
1977         my ($ds, $drive) = @_;
1978
1979         return if drive_is_cdrom($drive);
1980
1981         my $volid = $drive->{file};
1982
1983         return if !$volid || $volid =~ m|^/|;
1984
1985         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1986         return if !$path || !$owner || ($owner != $vmid);
1987
1988         PVE::Storage::vdisk_free($storecfg, $volid);
1989     });
1990
1991     if ($keep_empty_config) {
1992         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1993     } else {
1994         unlink $conffile;
1995     }
1996
1997     # also remove unused disk
1998     eval {
1999         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2000
2001         eval {
2002             PVE::Storage::foreach_volid($dl, sub {
2003                 my ($volid, $sid, $volname, $d) = @_;
2004                 PVE::Storage::vdisk_free($storecfg, $volid);
2005             });
2006         };
2007         warn $@ if $@;
2008
2009     };
2010     warn $@ if $@;
2011 }
2012
2013 sub parse_vm_config {
2014     my ($filename, $raw) = @_;
2015
2016     return undef if !defined($raw);
2017
2018     my $res = {
2019         digest => Digest::SHA::sha1_hex($raw),
2020         snapshots => {},
2021         pending => {},
2022     };
2023
2024     $filename =~ m|/qemu-server/(\d+)\.conf$|
2025         || die "got strange filename '$filename'";
2026
2027     my $vmid = $1;
2028
2029     my $conf = $res;
2030     my $descr;
2031     my $section = '';
2032
2033     my @lines = split(/\n/, $raw);
2034     foreach my $line (@lines) {
2035         next if $line =~ m/^\s*$/;
2036
2037         if ($line =~ m/^\[PENDING\]\s*$/i) {
2038             $section = 'pending';
2039             if (defined($descr)) {
2040                 $descr =~ s/\s+$//;
2041                 $conf->{description} = $descr;
2042             }
2043             $descr = undef;
2044             $conf = $res->{$section} = {};
2045             next;
2046
2047         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2048             $section = $1;
2049             if (defined($descr)) {
2050                 $descr =~ s/\s+$//;
2051                 $conf->{description} = $descr;
2052             }
2053             $descr = undef;
2054             $conf = $res->{snapshots}->{$section} = {};
2055             next;
2056         }
2057
2058         if ($line =~ m/^\#(.*)\s*$/) {
2059             $descr = '' if !defined($descr);
2060             $descr .= PVE::Tools::decode_text($1) . "\n";
2061             next;
2062         }
2063
2064         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2065             $descr = '' if !defined($descr);
2066             $descr .= PVE::Tools::decode_text($2);
2067         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2068             $conf->{snapstate} = $1;
2069         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2070             my $key = $1;
2071             my $value = $2;
2072             $conf->{$key} = $value;
2073         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2074             my $value = $1;
2075             if ($section eq 'pending') {
2076                 $conf->{delete} = $value; # we parse this later
2077             } else {
2078                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2079             }
2080         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2081             my $key = $1;
2082             my $value = $2;
2083             eval { $value = check_type($key, $value); };
2084             if ($@) {
2085                 warn "vm $vmid - unable to parse value of '$key' - $@";
2086             } else {
2087                 my $fmt = $confdesc->{$key}->{format};
2088                 if ($fmt && $fmt eq 'pve-qm-drive') {
2089                     my $v = parse_drive($key, $value);
2090                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2091                         $v->{file} = $volid;
2092                         $value = print_drive($vmid, $v);
2093                     } else {
2094                         warn "vm $vmid - unable to parse value of '$key'\n";
2095                         next;
2096                     }
2097                 }
2098
2099                 if ($key eq 'cdrom') {
2100                     $conf->{ide2} = $value;
2101                 } else {
2102                     $conf->{$key} = $value;
2103                 }
2104             }
2105         }
2106     }
2107
2108     if (defined($descr)) {
2109         $descr =~ s/\s+$//;
2110         $conf->{description} = $descr;
2111     }
2112     delete $res->{snapstate}; # just to be sure
2113
2114     return $res;
2115 }
2116
2117 sub write_vm_config {
2118     my ($filename, $conf) = @_;
2119
2120     delete $conf->{snapstate}; # just to be sure
2121
2122     if ($conf->{cdrom}) {
2123         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2124         $conf->{ide2} = $conf->{cdrom};
2125         delete $conf->{cdrom};
2126     }
2127
2128     # we do not use 'smp' any longer
2129     if ($conf->{sockets}) {
2130         delete $conf->{smp};
2131     } elsif ($conf->{smp}) {
2132         $conf->{sockets} = $conf->{smp};
2133         delete $conf->{cores};
2134         delete $conf->{smp};
2135     }
2136
2137     my $used_volids = {};
2138
2139     my $cleanup_config = sub {
2140         my ($cref, $pending, $snapname) = @_;
2141
2142         foreach my $key (keys %$cref) {
2143             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2144                 $key eq 'snapstate' || $key eq 'pending';
2145             my $value = $cref->{$key};
2146             if ($key eq 'delete') {
2147                 die "propertry 'delete' is only allowed in [PENDING]\n"
2148                     if !$pending;
2149                 # fixme: check syntax?
2150                 next;
2151             }
2152             eval { $value = check_type($key, $value); };
2153             die "unable to parse value of '$key' - $@" if $@;
2154
2155             $cref->{$key} = $value;
2156
2157             if (!$snapname && is_valid_drivename($key)) {
2158                 my $drive = parse_drive($key, $value);
2159                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2160             }
2161         }
2162     };
2163
2164     &$cleanup_config($conf);
2165
2166     &$cleanup_config($conf->{pending}, 1);
2167
2168     foreach my $snapname (keys %{$conf->{snapshots}}) {
2169         die "internal error" if $snapname eq 'pending';
2170         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2171     }
2172
2173     # remove 'unusedX' settings if we re-add a volume
2174     foreach my $key (keys %$conf) {
2175         my $value = $conf->{$key};
2176         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2177             delete $conf->{$key};
2178         }
2179     }
2180
2181     my $generate_raw_config = sub {
2182         my ($conf, $pending) = @_;
2183
2184         my $raw = '';
2185
2186         # add description as comment to top of file
2187         if (defined(my $descr = $conf->{description})) {
2188             if ($descr) {
2189                 foreach my $cl (split(/\n/, $descr)) {
2190                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2191                 }
2192             } else {
2193                 $raw .= "#\n" if $pending;
2194             }
2195         }
2196
2197         foreach my $key (sort keys %$conf) {
2198             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2199             $raw .= "$key: $conf->{$key}\n";
2200         }
2201         return $raw;
2202     };
2203
2204     my $raw = &$generate_raw_config($conf);
2205
2206     if (scalar(keys %{$conf->{pending}})){
2207         $raw .= "\n[PENDING]\n";
2208         $raw .= &$generate_raw_config($conf->{pending}, 1);
2209     }
2210
2211     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2212         $raw .= "\n[$snapname]\n";
2213         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2214     }
2215
2216     return $raw;
2217 }
2218
2219 sub load_defaults {
2220
2221     my $res = {};
2222
2223     # we use static defaults from our JSON schema configuration
2224     foreach my $key (keys %$confdesc) {
2225         if (defined(my $default = $confdesc->{$key}->{default})) {
2226             $res->{$key} = $default;
2227         }
2228     }
2229
2230     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2231     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2232
2233     return $res;
2234 }
2235
2236 sub config_list {
2237     my $vmlist = PVE::Cluster::get_vmlist();
2238     my $res = {};
2239     return $res if !$vmlist || !$vmlist->{ids};
2240     my $ids = $vmlist->{ids};
2241
2242     foreach my $vmid (keys %$ids) {
2243         my $d = $ids->{$vmid};
2244         next if !$d->{node} || $d->{node} ne $nodename;
2245         next if !$d->{type} || $d->{type} ne 'qemu';
2246         $res->{$vmid}->{exists} = 1;
2247     }
2248     return $res;
2249 }
2250
2251 # test if VM uses local resources (to prevent migration)
2252 sub check_local_resources {
2253     my ($conf, $noerr) = @_;
2254
2255     my $loc_res = 0;
2256
2257     $loc_res = 1 if $conf->{hostusb}; # old syntax
2258     $loc_res = 1 if $conf->{hostpci}; # old syntax
2259
2260     foreach my $k (keys %$conf) {
2261         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2262         # sockets are safe: they will recreated be on the target side post-migrate
2263         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2264         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2265     }
2266
2267     die "VM uses local resources\n" if $loc_res && !$noerr;
2268
2269     return $loc_res;
2270 }
2271
2272 # check if used storages are available on all nodes (use by migrate)
2273 sub check_storage_availability {
2274     my ($storecfg, $conf, $node) = @_;
2275
2276     foreach_drive($conf, sub {
2277         my ($ds, $drive) = @_;
2278
2279         my $volid = $drive->{file};
2280         return if !$volid;
2281
2282         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2283         return if !$sid;
2284
2285         # check if storage is available on both nodes
2286         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2287         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2288    });
2289 }
2290
2291 # list nodes where all VM images are available (used by has_feature API)
2292 sub shared_nodes {
2293     my ($conf, $storecfg) = @_;
2294
2295     my $nodelist = PVE::Cluster::get_nodelist();
2296     my $nodehash = { map { $_ => 1 } @$nodelist };
2297     my $nodename = PVE::INotify::nodename();
2298
2299     foreach_drive($conf, sub {
2300         my ($ds, $drive) = @_;
2301
2302         my $volid = $drive->{file};
2303         return if !$volid;
2304
2305         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2306         if ($storeid) {
2307             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2308             if ($scfg->{disable}) {
2309                 $nodehash = {};
2310             } elsif (my $avail = $scfg->{nodes}) {
2311                 foreach my $node (keys %$nodehash) {
2312                     delete $nodehash->{$node} if !$avail->{$node};
2313                 }
2314             } elsif (!$scfg->{shared}) {
2315                 foreach my $node (keys %$nodehash) {
2316                     delete $nodehash->{$node} if $node ne $nodename
2317                 }
2318             }
2319         }
2320     });
2321
2322     return $nodehash
2323 }
2324
2325 sub check_cmdline {
2326     my ($pidfile, $pid) = @_;
2327
2328     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2329     if (defined($fh)) {
2330         my $line = <$fh>;
2331         $fh->close;
2332         return undef if !$line;
2333         my @param = split(/\0/, $line);
2334
2335         my $cmd = $param[0];
2336         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2337
2338         for (my $i = 0; $i < scalar (@param); $i++) {
2339             my $p = $param[$i];
2340             next if !$p;
2341             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2342                 my $p = $param[$i+1];
2343                 return 1 if $p && ($p eq $pidfile);
2344                 return undef;
2345             }
2346         }
2347     }
2348     return undef;
2349 }
2350
2351 sub check_running {
2352     my ($vmid, $nocheck, $node) = @_;
2353
2354     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2355
2356     die "unable to find configuration file for VM $vmid - no such machine\n"
2357         if !$nocheck && ! -f $filename;
2358
2359     my $pidfile = pidfile_name($vmid);
2360
2361     if (my $fd = IO::File->new("<$pidfile")) {
2362         my $st = stat($fd);
2363         my $line = <$fd>;
2364         close($fd);
2365
2366         my $mtime = $st->mtime;
2367         if ($mtime > time()) {
2368             warn "file '$filename' modified in future\n";
2369         }
2370
2371         if ($line =~ m/^(\d+)$/) {
2372             my $pid = $1;
2373             if (check_cmdline($pidfile, $pid)) {
2374                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2375                     return $pid;
2376                 }
2377             }
2378         }
2379     }
2380
2381     return undef;
2382 }
2383
2384 sub vzlist {
2385
2386     my $vzlist = config_list();
2387
2388     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2389
2390     while (defined(my $de = $fd->read)) {
2391         next if $de !~ m/^(\d+)\.pid$/;
2392         my $vmid = $1;
2393         next if !defined($vzlist->{$vmid});
2394         if (my $pid = check_running($vmid)) {
2395             $vzlist->{$vmid}->{pid} = $pid;
2396         }
2397     }
2398
2399     return $vzlist;
2400 }
2401
2402 sub disksize {
2403     my ($storecfg, $conf) = @_;
2404
2405     my $bootdisk = $conf->{bootdisk};
2406     return undef if !$bootdisk;
2407     return undef if !is_valid_drivename($bootdisk);
2408
2409     return undef if !$conf->{$bootdisk};
2410
2411     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2412     return undef if !defined($drive);
2413
2414     return undef if drive_is_cdrom($drive);
2415
2416     my $volid = $drive->{file};
2417     return undef if !$volid;
2418
2419     return $drive->{size};
2420 }
2421
2422 my $last_proc_pid_stat;
2423
2424 # get VM status information
2425 # This must be fast and should not block ($full == false)
2426 # We only query KVM using QMP if $full == true (this can be slow)
2427 sub vmstatus {
2428     my ($opt_vmid, $full) = @_;
2429
2430     my $res = {};
2431
2432     my $storecfg = PVE::Storage::config();
2433
2434     my $list = vzlist();
2435     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2436
2437     my $cpucount = $cpuinfo->{cpus} || 1;
2438
2439     foreach my $vmid (keys %$list) {
2440         next if $opt_vmid && ($vmid ne $opt_vmid);
2441
2442         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2443         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2444
2445         my $d = {};
2446         $d->{pid} = $list->{$vmid}->{pid};
2447
2448         # fixme: better status?
2449         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2450
2451         my $size = disksize($storecfg, $conf);
2452         if (defined($size)) {
2453             $d->{disk} = 0; # no info available
2454             $d->{maxdisk} = $size;
2455         } else {
2456             $d->{disk} = 0;
2457             $d->{maxdisk} = 0;
2458         }
2459
2460         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2461         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2462         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2463
2464         $d->{name} = $conf->{name} || "VM $vmid";
2465         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2466
2467         if ($conf->{balloon}) {
2468             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2469             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2470         }
2471
2472         $d->{uptime} = 0;
2473         $d->{cpu} = 0;
2474         $d->{mem} = 0;
2475
2476         $d->{netout} = 0;
2477         $d->{netin} = 0;
2478
2479         $d->{diskread} = 0;
2480         $d->{diskwrite} = 0;
2481
2482         $d->{template} = PVE::QemuConfig->is_template($conf);
2483
2484         $res->{$vmid} = $d;
2485     }
2486
2487     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2488     foreach my $dev (keys %$netdev) {
2489         next if $dev !~ m/^tap([1-9]\d*)i/;
2490         my $vmid = $1;
2491         my $d = $res->{$vmid};
2492         next if !$d;
2493
2494         $d->{netout} += $netdev->{$dev}->{receive};
2495         $d->{netin} += $netdev->{$dev}->{transmit};
2496
2497         if ($full) {
2498             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2499             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2500         }
2501
2502     }
2503
2504     my $ctime = gettimeofday;
2505
2506     foreach my $vmid (keys %$list) {
2507
2508         my $d = $res->{$vmid};
2509         my $pid = $d->{pid};
2510         next if !$pid;
2511
2512         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2513         next if !$pstat; # not running
2514
2515         my $used = $pstat->{utime} + $pstat->{stime};
2516
2517         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2518
2519         if ($pstat->{vsize}) {
2520             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2521         }
2522
2523         my $old = $last_proc_pid_stat->{$pid};
2524         if (!$old) {
2525             $last_proc_pid_stat->{$pid} = {
2526                 time => $ctime,
2527                 used => $used,
2528                 cpu => 0,
2529             };
2530             next;
2531         }
2532
2533         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2534
2535         if ($dtime > 1000) {
2536             my $dutime = $used -  $old->{used};
2537
2538             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2539             $last_proc_pid_stat->{$pid} = {
2540                 time => $ctime,
2541                 used => $used,
2542                 cpu => $d->{cpu},
2543             };
2544         } else {
2545             $d->{cpu} = $old->{cpu};
2546         }
2547     }
2548
2549     return $res if !$full;
2550
2551     my $qmpclient = PVE::QMPClient->new();
2552
2553     my $ballooncb = sub {
2554         my ($vmid, $resp) = @_;
2555
2556         my $info = $resp->{'return'};
2557         return if !$info->{max_mem};
2558
2559         my $d = $res->{$vmid};
2560
2561         # use memory assigned to VM
2562         $d->{maxmem} = $info->{max_mem};
2563         $d->{balloon} = $info->{actual};
2564
2565         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2566             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2567             $d->{freemem} = $info->{free_mem};
2568         }
2569
2570         $d->{ballooninfo} = $info;
2571     };
2572
2573     my $blockstatscb = sub {
2574         my ($vmid, $resp) = @_;
2575         my $data = $resp->{'return'} || [];
2576         my $totalrdbytes = 0;
2577         my $totalwrbytes = 0;
2578
2579         for my $blockstat (@$data) {
2580             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2581             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2582
2583             $blockstat->{device} =~ s/drive-//;
2584             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2585         }
2586         $res->{$vmid}->{diskread} = $totalrdbytes;
2587         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2588     };
2589
2590     my $statuscb = sub {
2591         my ($vmid, $resp) = @_;
2592
2593         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2594         # this fails if ballon driver is not loaded, so this must be
2595         # the last commnand (following command are aborted if this fails).
2596         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2597
2598         my $status = 'unknown';
2599         if (!defined($status = $resp->{'return'}->{status})) {
2600             warn "unable to get VM status\n";
2601             return;
2602         }
2603
2604         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2605     };
2606
2607     foreach my $vmid (keys %$list) {
2608         next if $opt_vmid && ($vmid ne $opt_vmid);
2609         next if !$res->{$vmid}->{pid}; # not running
2610         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2611     }
2612
2613     $qmpclient->queue_execute(undef, 1);
2614
2615     foreach my $vmid (keys %$list) {
2616         next if $opt_vmid && ($vmid ne $opt_vmid);
2617         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2618     }
2619
2620     return $res;
2621 }
2622
2623 sub foreach_dimm {
2624     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2625
2626     my $dimm_id = 0;
2627     my $current_size = 1024;
2628     my $dimm_size = 512;
2629     return if $current_size == $memory;
2630
2631     for (my $j = 0; $j < 8; $j++) {
2632         for (my $i = 0; $i < 32; $i++) {
2633             my $name = "dimm${dimm_id}";
2634             $dimm_id++;
2635             my $numanode = $i % $sockets;
2636             $current_size += $dimm_size;
2637             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2638             return  $current_size if $current_size >= $memory;
2639         }
2640         $dimm_size *= 2;
2641     }
2642 }
2643
2644 sub foreach_reverse_dimm {
2645     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2646
2647     my $dimm_id = 253;
2648     my $current_size = 4177920;
2649     my $dimm_size = 65536;
2650     return if $current_size == $memory;
2651
2652     for (my $j = 0; $j < 8; $j++) {
2653         for (my $i = 0; $i < 32; $i++) {
2654             my $name = "dimm${dimm_id}";
2655             $dimm_id--;
2656             my $numanode = $i % $sockets;
2657             $current_size -= $dimm_size;
2658             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2659             return  $current_size if $current_size <= $memory;
2660         }
2661         $dimm_size /= 2;
2662     }
2663 }
2664
2665 sub foreach_drive {
2666     my ($conf, $func) = @_;
2667
2668     foreach my $ds (valid_drive_names()) {
2669         next if !defined($conf->{$ds});
2670
2671         my $drive = parse_drive($ds, $conf->{$ds});
2672         next if !$drive;
2673
2674         &$func($ds, $drive);
2675     }
2676 }
2677
2678 sub foreach_volid {
2679     my ($conf, $func) = @_;
2680
2681     my $volhash = {};
2682
2683     my $test_volid = sub {
2684         my ($volid, $is_cdrom) = @_;
2685
2686         return if !$volid;
2687
2688         $volhash->{$volid} = $is_cdrom || 0;
2689     };
2690
2691     foreach_drive($conf, sub {
2692         my ($ds, $drive) = @_;
2693         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2694     });
2695
2696     foreach my $snapname (keys %{$conf->{snapshots}}) {
2697         my $snap = $conf->{snapshots}->{$snapname};
2698         &$test_volid($snap->{vmstate}, 0);
2699         foreach_drive($snap, sub {
2700             my ($ds, $drive) = @_;
2701             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2702         });
2703     }
2704
2705     foreach my $volid (keys %$volhash) {
2706         &$func($volid, $volhash->{$volid});
2707     }
2708 }
2709
2710 sub vga_conf_has_spice {
2711     my ($vga) = @_;
2712
2713     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2714
2715     return $1 || 1;
2716 }
2717
2718 sub config_to_command {
2719     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2720
2721     my $cmd = [];
2722     my $globalFlags = [];
2723     my $machineFlags = [];
2724     my $rtcFlags = [];
2725     my $cpuFlags = [];
2726     my $devices = [];
2727     my $pciaddr = '';
2728     my $bridges = {};
2729     my $kvmver = kvm_user_version();
2730     my $vernum = 0; # unknown
2731     my $ostype = $conf->{ostype};
2732     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2733         $vernum = $1*1000000+$2*1000;
2734     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2735         $vernum = $1*1000000+$2*1000+$3;
2736     }
2737
2738     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2739
2740     my $have_ovz = -f '/proc/vz/vestat';
2741
2742     my $q35 = machine_type_is_q35($conf);
2743     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2744     my $machine_type = $forcemachine || $conf->{machine};
2745     my $use_old_bios_files = undef;
2746     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2747
2748     my $cpuunits = defined($conf->{cpuunits}) ?
2749             $conf->{cpuunits} : $defaults->{cpuunits};
2750
2751     push @$cmd, '/usr/bin/systemd-run';
2752     push @$cmd, '--scope';
2753     push @$cmd, '--slice', "qemu";
2754     push @$cmd, '--unit', $vmid;
2755     # set KillMode=none, so that systemd don't kill those scopes
2756     # at shutdown (pve-manager service should stop the VMs instead)
2757     push @$cmd, '-p', "KillMode=none";
2758     push @$cmd, '-p', "CPUShares=$cpuunits";
2759     if ($conf->{cpulimit}) {
2760         my $cpulimit = int($conf->{cpulimit} * 100);
2761         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2762     }
2763
2764     push @$cmd, '/usr/bin/kvm';
2765
2766     push @$cmd, '-id', $vmid;
2767
2768     my $use_virtio = 0;
2769
2770     my $qmpsocket = qmp_socket($vmid);
2771     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2772     push @$cmd, '-mon', "chardev=qmp,mode=control";
2773
2774
2775     push @$cmd, '-pidfile' , pidfile_name($vmid);
2776
2777     push @$cmd, '-daemonize';
2778
2779     if ($conf->{smbios1}) {
2780         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2781     }
2782
2783     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2784         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2785         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2786         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2787         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2788         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2789         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2790     }
2791
2792     if ($q35) {
2793         # the q35 chipset support native usb2, so we enable usb controller
2794         # by default for this machine type
2795         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2796     } else {
2797         $pciaddr = print_pci_addr("piix3", $bridges);
2798         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2799
2800         my $use_usb2 = 0;
2801         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2802             next if !$conf->{"usb$i"};
2803             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2804             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2805             $use_usb2 = 1;
2806         }
2807         # include usb device config
2808         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2809     }
2810
2811     # add usb3 controller if needed
2812
2813     my $use_usb3 = 0;
2814     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2815         next if !$conf->{"usb$i"};
2816         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2817         next if !$d || !$d->{usb3};
2818         $use_usb3 = 1;
2819     }
2820
2821     $pciaddr = print_pci_addr("xhci", $bridges);
2822     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2823
2824     my $vga = $conf->{vga};
2825
2826     my $qxlnum = vga_conf_has_spice($vga);
2827     $vga = 'qxl' if $qxlnum;
2828
2829     if (!$vga) {
2830         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2831                                 $conf->{ostype} eq 'win7' ||
2832                                 $conf->{ostype} eq 'w2k8')) {
2833             $vga = 'std';
2834         } else {
2835             $vga = 'cirrus';
2836         }
2837     }
2838
2839     # enable absolute mouse coordinates (needed by vnc)
2840     my $tablet;
2841     if (defined($conf->{tablet})) {
2842         $tablet = $conf->{tablet};
2843     } else {
2844         $tablet = $defaults->{tablet};
2845         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2846         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2847     }
2848
2849     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2850
2851     my $kvm_off = 0;
2852     # host pci devices
2853     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2854         my $d = parse_hostpci($conf->{"hostpci$i"});
2855         next if !$d;
2856
2857         my $pcie = $d->{pcie};
2858         if($pcie){
2859             die "q35 machine model is not enabled" if !$q35;
2860             $pciaddr = print_pcie_addr("hostpci$i");
2861         }else{
2862             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2863         }
2864
2865         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2866         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2867         if ($xvga && $xvga ne '') {
2868             $kvm_off = 1;
2869             $vga = 'none';
2870             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2871                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2872             }
2873             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2874                 $xvga = "";
2875             }
2876         }
2877         my $pcidevices = $d->{pciid};
2878         my $multifunction = 1 if @$pcidevices > 1;
2879
2880         my $j=0;
2881         foreach my $pcidevice (@$pcidevices) {
2882
2883             my $id = "hostpci$i";
2884             $id .= ".$j" if $multifunction;
2885             my $addr = $pciaddr;
2886             $addr .= ".$j" if $multifunction;
2887             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2888
2889             if($j == 0){
2890                 $devicestr .= "$rombar$xvga";
2891                 $devicestr .= ",multifunction=on" if $multifunction;
2892             }
2893
2894             push @$devices, '-device', $devicestr;
2895             $j++;
2896         }
2897     }
2898
2899     # usb devices
2900     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2901         next if !$conf->{"usb$i"};
2902         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2903         next if !$d;
2904
2905         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2906         my $usbbus = '';
2907         if (defined($d->{usb3}) && $d->{usb3}) {
2908             $usbbus = ',bus=xhci.0';
2909         }
2910
2911         if (defined($d->{host})) {
2912             $d = parse_usb_device($d->{host});
2913             if (defined($d->{vendorid}) && defined($d->{productid})) {
2914                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2915             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2916                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2917             } elsif (defined($d->{spice}) && $d->{spice}) {
2918                 # usb redir support for spice, currently no usb3
2919                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2920                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2921             }
2922         }
2923     }
2924
2925     # serial devices
2926     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2927         if (my $path = $conf->{"serial$i"}) {
2928             if ($path eq 'socket') {
2929                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2930                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2931                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2932             } else {
2933                 die "no such serial device\n" if ! -c $path;
2934                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2935                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2936             }
2937         }
2938     }
2939
2940     # parallel devices
2941     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2942         if (my $path = $conf->{"parallel$i"}) {
2943             die "no such parallel device\n" if ! -c $path;
2944             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2945             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2946             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2947         }
2948     }
2949
2950     my $vmname = $conf->{name} || "vm$vmid";
2951
2952     push @$cmd, '-name', $vmname;
2953
2954     my $sockets = 1;
2955     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2956     $sockets = $conf->{sockets} if  $conf->{sockets};
2957
2958     my $cores = $conf->{cores} || 1;
2959
2960     my $maxcpus = $sockets * $cores;
2961
2962     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2963
2964     my $allowed_vcpus = $cpuinfo->{cpus};
2965
2966     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2967         if ($allowed_vcpus < $maxcpus);
2968
2969     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2970
2971     push @$cmd, '-nodefaults';
2972
2973     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2974
2975     my $bootindex_hash = {};
2976     my $i = 1;
2977     foreach my $o (split(//, $bootorder)) {
2978         $bootindex_hash->{$o} = $i*100;
2979         $i++;
2980     }
2981
2982     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2983
2984     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2985
2986     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2987
2988     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2989
2990     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2991         my $socket = vnc_socket($vmid);
2992         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2993     } else {
2994         push @$cmd, '-nographic';
2995     }
2996
2997     # time drift fix
2998     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2999
3000     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3001     my $useLocaltime = $conf->{localtime};
3002
3003     if ($ostype) {
3004         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3005
3006         if ($ostype =~ m/^w/) { # windows
3007             $useLocaltime = 1 if !defined($conf->{localtime});
3008
3009             # use time drift fix when acpi is enabled
3010             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3011                 $tdf = 1 if !defined($conf->{tdf});
3012             }
3013         }
3014
3015         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3016             $ostype eq 'wvista') {
3017             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3018             push @$cmd, '-no-hpet';
3019             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3020                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3021                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3022                 push @$cpuFlags , 'hv_time' if !$nokvm;
3023
3024             } else {
3025                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3026             }
3027         }
3028
3029         if ($ostype eq 'win7' || $ostype eq 'win8') {
3030             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3031         }
3032     }
3033
3034     push @$rtcFlags, 'driftfix=slew' if $tdf;
3035
3036     if ($nokvm) {
3037         push @$machineFlags, 'accel=tcg';
3038     } else {
3039         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3040     }
3041
3042     if ($machine_type) {
3043         push @$machineFlags, "type=${machine_type}";
3044     }
3045
3046     if ($conf->{startdate}) {
3047         push @$rtcFlags, "base=$conf->{startdate}";
3048     } elsif ($useLocaltime) {
3049         push @$rtcFlags, 'base=localtime';
3050     }
3051
3052     my $cpu = $nokvm ? "qemu64" : "kvm64";
3053     if (my $cputype = $conf->{cpu}) {
3054         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3055             or die "Cannot parse cpu description: $cputype\n";
3056         $cpu = $cpuconf->{cputype};
3057         $kvm_off = 1 if $cpuconf->{hidden};
3058     }
3059
3060     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3061
3062     push @$cpuFlags , '-x2apic'
3063         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3064
3065     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3066
3067     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3068
3069     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3070
3071         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3072         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3073     }
3074
3075     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3076
3077     push @$cpuFlags, 'kvm=off' if $kvm_off;
3078
3079     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3080
3081     push @$cmd, '-cpu', $cpu;
3082
3083     my $memory = $conf->{memory} || $defaults->{memory};
3084     my $static_memory = 0;
3085     my $dimm_memory = 0;
3086
3087     if ($hotplug_features->{memory}) {
3088         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3089         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3090         $static_memory = $STATICMEM;
3091         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3092         $dimm_memory = $memory - $static_memory;
3093         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3094
3095     } else {
3096
3097         $static_memory = $memory;
3098         push @$cmd, '-m', $static_memory;
3099     }
3100
3101     if ($conf->{numa}) {
3102
3103         my $numa_totalmemory = undef;
3104         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3105             next if !$conf->{"numa$i"};
3106             my $numa = parse_numa($conf->{"numa$i"});
3107             next if !$numa;
3108             # memory
3109             die "missing numa node$i memory value\n" if !$numa->{memory};
3110             my $numa_memory = $numa->{memory};
3111             $numa_totalmemory += $numa_memory;
3112             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3113
3114             # cpus
3115             my $cpulists = $numa->{cpus};
3116             die "missing numa node$i cpus\n" if !defined($cpulists);
3117             my $cpus = join(',', map {
3118                 my ($start, $end) = @$_;
3119                 defined($end) ? "$start-$end" : $start
3120             } @$cpulists);
3121
3122             # hostnodes
3123             my $hostnodelists = $numa->{hostnodes};
3124             if (defined($hostnodelists)) {
3125                 my $hostnodes;
3126                 foreach my $hostnoderange (@$hostnodelists) {
3127                     my ($start, $end) = @$hostnoderange;
3128                     $hostnodes .= ',' if $hostnodes;
3129                     $hostnodes .= $start;
3130                     $hostnodes .= "-$end" if defined($end);
3131                     $end //= $start;
3132                     for (my $i = $start; $i <= $end; ++$i ) {
3133                         die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3134                     }
3135                 }
3136
3137                 # policy
3138                 my $policy = $numa->{policy};
3139                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3140                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3141             }
3142
3143             push @$cmd, '-object', $numa_object;
3144             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3145         }
3146
3147         die "total memory for NUMA nodes must be equal to vm static memory\n"
3148             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3149
3150         #if no custom tology, we split memory and cores across numa nodes
3151         if(!$numa_totalmemory) {
3152
3153             my $numa_memory = ($static_memory / $sockets) . "M";
3154
3155             for (my $i = 0; $i < $sockets; $i++)  {
3156
3157                 my $cpustart = ($cores * $i);
3158                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3159                 my $cpus = $cpustart;
3160                 $cpus .= "-$cpuend" if $cpuend;
3161
3162                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3163                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3164             }
3165         }
3166     }
3167
3168     if ($hotplug_features->{memory}) {
3169         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3170             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3171             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3172             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3173
3174             #if dimm_memory is not aligned to dimm map
3175             if($current_size > $memory) {
3176                  $conf->{memory} = $current_size;
3177                  PVE::QemuConfig->write_config($vmid, $conf);
3178             }
3179         });
3180     }
3181
3182     push @$cmd, '-S' if $conf->{freeze};
3183
3184     # set keyboard layout
3185     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3186     push @$cmd, '-k', $kb if $kb;
3187
3188     # enable sound
3189     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3190     #push @$cmd, '-soundhw', 'es1370';
3191     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3192
3193     if($conf->{agent}) {
3194         my $qgasocket = qmp_socket($vmid, 1);
3195         my $pciaddr = print_pci_addr("qga0", $bridges);
3196         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3197         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3198         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3199     }
3200
3201     my $spice_port;
3202
3203     if ($qxlnum) {
3204         if ($qxlnum > 1) {
3205             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3206                 for(my $i = 1; $i < $qxlnum; $i++){
3207                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3208                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3209                 }
3210             } else {
3211                 # assume other OS works like Linux
3212                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3213                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3214             }
3215         }
3216
3217         my $pciaddr = print_pci_addr("spice", $bridges);
3218
3219         my $nodename = PVE::INotify::nodename();
3220         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3221         $spice_port = PVE::Tools::next_spice_port($pfamily);
3222
3223         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3224
3225         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3226         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3227         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3228     }
3229
3230     # enable balloon by default, unless explicitly disabled
3231     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3232         $pciaddr = print_pci_addr("balloon0", $bridges);
3233         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3234     }
3235
3236     if ($conf->{watchdog}) {
3237         my $wdopts = parse_watchdog($conf->{watchdog});
3238         $pciaddr = print_pci_addr("watchdog", $bridges);
3239         my $watchdog = $wdopts->{model} || 'i6300esb';
3240         push @$devices, '-device', "$watchdog$pciaddr";
3241         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3242     }
3243
3244     my $vollist = [];
3245     my $scsicontroller = {};
3246     my $ahcicontroller = {};
3247     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3248
3249     # Add iscsi initiator name if available
3250     if (my $initiator = get_initiator_name()) {
3251         push @$devices, '-iscsi', "initiator-name=$initiator";
3252     }
3253
3254     foreach_drive($conf, sub {
3255         my ($ds, $drive) = @_;
3256
3257         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3258             push @$vollist, $drive->{file};
3259         }
3260
3261         $use_virtio = 1 if $ds =~ m/^virtio/;
3262
3263         if (drive_is_cdrom ($drive)) {
3264             if ($bootindex_hash->{d}) {
3265                 $drive->{bootindex} = $bootindex_hash->{d};
3266                 $bootindex_hash->{d} += 1;
3267             }
3268         } else {
3269             if ($bootindex_hash->{c}) {
3270                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3271                 $bootindex_hash->{c} += 1;
3272             }
3273         }
3274
3275         if($drive->{interface} eq 'virtio'){
3276            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3277         }
3278
3279         if ($drive->{interface} eq 'scsi') {
3280
3281             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3282
3283             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3284             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3285
3286             my $iothread = '';
3287             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3288                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3289                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3290             }
3291
3292             my $queues = '';
3293             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3294                 $queues = ",num_queues=$drive->{queues}";
3295             }
3296
3297             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3298             $scsicontroller->{$controller}=1;
3299         }
3300
3301         if ($drive->{interface} eq 'sata') {
3302            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3303            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3304            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3305            $ahcicontroller->{$controller}=1;
3306         }
3307
3308         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3309         push @$devices, '-drive',$drive_cmd;
3310         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3311     });
3312
3313     for (my $i = 0; $i < $MAX_NETS; $i++) {
3314          next if !$conf->{"net$i"};
3315          my $d = parse_net($conf->{"net$i"});
3316          next if !$d;
3317
3318          $use_virtio = 1 if $d->{model} eq 'virtio';
3319
3320          if ($bootindex_hash->{n}) {
3321             $d->{bootindex} = $bootindex_hash->{n};
3322             $bootindex_hash->{n} += 1;
3323          }
3324
3325          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3326          push @$devices, '-netdev', $netdevfull;
3327
3328          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3329          push @$devices, '-device', $netdevicefull;
3330     }
3331
3332     if (!$q35) {
3333         # add pci bridges
3334         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3335            $bridges->{1} = 1;
3336            $bridges->{2} = 1;
3337         }
3338
3339         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3340
3341         while (my ($k, $v) = each %$bridges) {
3342             $pciaddr = print_pci_addr("pci.$k");
3343             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3344         }
3345     }
3346
3347     # add custom args
3348     if ($conf->{args}) {
3349         my $aa = PVE::Tools::split_args($conf->{args});
3350         push @$cmd, @$aa;
3351     }
3352
3353     push @$cmd, @$devices;
3354     push @$cmd, '-rtc', join(',', @$rtcFlags)
3355         if scalar(@$rtcFlags);
3356     push @$cmd, '-machine', join(',', @$machineFlags)
3357         if scalar(@$machineFlags);
3358     push @$cmd, '-global', join(',', @$globalFlags)
3359         if scalar(@$globalFlags);
3360
3361     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3362 }
3363
3364 sub vnc_socket {
3365     my ($vmid) = @_;
3366     return "${var_run_tmpdir}/$vmid.vnc";
3367 }
3368
3369 sub spice_port {
3370     my ($vmid) = @_;
3371
3372     my $res = vm_mon_cmd($vmid, 'query-spice');
3373
3374     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3375 }
3376
3377 sub qmp_socket {
3378     my ($vmid, $qga) = @_;
3379     my $sockettype = $qga ? 'qga' : 'qmp';
3380     return "${var_run_tmpdir}/$vmid.$sockettype";
3381 }
3382
3383 sub pidfile_name {
3384     my ($vmid) = @_;
3385     return "${var_run_tmpdir}/$vmid.pid";
3386 }
3387
3388 sub vm_devices_list {
3389     my ($vmid) = @_;
3390
3391     my $res = vm_mon_cmd($vmid, 'query-pci');
3392     my $devices = {};
3393     foreach my $pcibus (@$res) {
3394         foreach my $device (@{$pcibus->{devices}}) {
3395             next if !$device->{'qdev_id'};
3396             if ($device->{'pci_bridge'}) {
3397                 $devices->{$device->{'qdev_id'}} = 1;
3398                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3399                     next if !$bridge_device->{'qdev_id'};
3400                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3401                     $devices->{$device->{'qdev_id'}}++;
3402                 }
3403             } else {
3404                 $devices->{$device->{'qdev_id'}} = 1;
3405             }
3406         }
3407     }
3408
3409     my $resblock = vm_mon_cmd($vmid, 'query-block');
3410     foreach my $block (@$resblock) {
3411         if($block->{device} =~ m/^drive-(\S+)/){
3412                 $devices->{$1} = 1;
3413         }
3414     }
3415
3416     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3417     foreach my $mice (@$resmice) {
3418         if ($mice->{name} eq 'QEMU HID Tablet') {
3419             $devices->{tablet} = 1;
3420             last;
3421         }
3422     }
3423
3424     return $devices;
3425 }
3426
3427 sub vm_deviceplug {
3428     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3429
3430     my $q35 = machine_type_is_q35($conf);
3431
3432     my $devices_list = vm_devices_list($vmid);
3433     return 1 if defined($devices_list->{$deviceid});
3434
3435     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3436
3437     if ($deviceid eq 'tablet') {
3438
3439         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3440
3441     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3442
3443         qemu_iothread_add($vmid, $deviceid, $device);
3444
3445         qemu_driveadd($storecfg, $vmid, $device);
3446         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3447
3448         qemu_deviceadd($vmid, $devicefull);
3449         eval { qemu_deviceaddverify($vmid, $deviceid); };
3450         if (my $err = $@) {
3451             eval { qemu_drivedel($vmid, $deviceid); };
3452             warn $@ if $@;
3453             die $err;
3454         }
3455
3456     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3457
3458
3459         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3460         my $pciaddr = print_pci_addr($deviceid);
3461         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3462
3463         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3464
3465         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3466             qemu_iothread_add($vmid, $deviceid, $device);
3467             $devicefull .= ",iothread=iothread-$deviceid";
3468         }
3469
3470         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3471             $devicefull .= ",num_queues=$device->{queues}";
3472         }
3473
3474         qemu_deviceadd($vmid, $devicefull);
3475         qemu_deviceaddverify($vmid, $deviceid);
3476
3477     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3478
3479         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3480         qemu_driveadd($storecfg, $vmid, $device);
3481
3482         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3483         eval { qemu_deviceadd($vmid, $devicefull); };
3484         if (my $err = $@) {
3485             eval { qemu_drivedel($vmid, $deviceid); };
3486             warn $@ if $@;
3487             die $err;
3488         }
3489
3490     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3491
3492         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3493
3494         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3495         my $use_old_bios_files = undef;
3496         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3497
3498         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3499         qemu_deviceadd($vmid, $netdevicefull);
3500         eval { qemu_deviceaddverify($vmid, $deviceid); };
3501         if (my $err = $@) {
3502             eval { qemu_netdevdel($vmid, $deviceid); };
3503             warn $@ if $@;
3504             die $err;
3505         }
3506
3507     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3508
3509         my $bridgeid = $2;
3510         my $pciaddr = print_pci_addr($deviceid);
3511         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3512
3513         qemu_deviceadd($vmid, $devicefull);
3514         qemu_deviceaddverify($vmid, $deviceid);
3515
3516     } else {
3517         die "can't hotplug device '$deviceid'\n";
3518     }
3519
3520     return 1;
3521 }
3522
3523 # fixme: this should raise exceptions on error!
3524 sub vm_deviceunplug {
3525     my ($vmid, $conf, $deviceid) = @_;
3526
3527     my $devices_list = vm_devices_list($vmid);
3528     return 1 if !defined($devices_list->{$deviceid});
3529
3530     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3531
3532     if ($deviceid eq 'tablet') {
3533
3534         qemu_devicedel($vmid, $deviceid);
3535
3536     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3537
3538         qemu_devicedel($vmid, $deviceid);
3539         qemu_devicedelverify($vmid, $deviceid);
3540         qemu_drivedel($vmid, $deviceid);
3541         qemu_iothread_del($conf, $vmid, $deviceid);
3542
3543     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3544
3545         qemu_devicedel($vmid, $deviceid);
3546         qemu_devicedelverify($vmid, $deviceid);
3547         qemu_iothread_del($conf, $vmid, $deviceid);
3548
3549     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3550
3551         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3552         my $device = parse_drive($deviceid, $conf->{$deviceid});
3553         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3554
3555         qemu_devicedel($vmid, $deviceid);
3556         qemu_drivedel($vmid, $deviceid);
3557         qemu_deletescsihw($conf, $vmid, $deviceid);
3558
3559     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3560
3561         qemu_devicedel($vmid, $deviceid);
3562         qemu_devicedelverify($vmid, $deviceid);
3563         qemu_netdevdel($vmid, $deviceid);
3564
3565     } else {
3566         die "can't unplug device '$deviceid'\n";
3567     }
3568
3569     return 1;
3570 }
3571
3572 sub qemu_deviceadd {
3573     my ($vmid, $devicefull) = @_;
3574
3575     $devicefull = "driver=".$devicefull;
3576     my %options =  split(/[=,]/, $devicefull);
3577
3578     vm_mon_cmd($vmid, "device_add" , %options);
3579 }
3580
3581 sub qemu_devicedel {
3582     my ($vmid, $deviceid) = @_;
3583
3584     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3585 }
3586
3587 sub qemu_iothread_add {
3588     my($vmid, $deviceid, $device) = @_;
3589
3590     if ($device->{iothread}) {
3591         my $iothreads = vm_iothreads_list($vmid);
3592         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3593     }
3594 }
3595
3596 sub qemu_iothread_del {
3597     my($conf, $vmid, $deviceid) = @_;
3598
3599     my $device = parse_drive($deviceid, $conf->{$deviceid});
3600     if ($device->{iothread}) {
3601         my $iothreads = vm_iothreads_list($vmid);
3602         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3603     }
3604 }
3605
3606 sub qemu_objectadd {
3607     my($vmid, $objectid, $qomtype) = @_;
3608
3609     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3610
3611     return 1;
3612 }
3613
3614 sub qemu_objectdel {
3615     my($vmid, $objectid) = @_;
3616
3617     vm_mon_cmd($vmid, "object-del", id => $objectid);
3618
3619     return 1;
3620 }
3621
3622 sub qemu_driveadd {
3623     my ($storecfg, $vmid, $device) = @_;
3624
3625     my $drive = print_drive_full($storecfg, $vmid, $device);
3626     $drive =~ s/\\/\\\\/g;
3627     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3628
3629     # If the command succeeds qemu prints: "OK"
3630     return 1 if $ret =~ m/OK/s;
3631
3632     die "adding drive failed: $ret\n";
3633 }
3634
3635 sub qemu_drivedel {
3636     my($vmid, $deviceid) = @_;
3637
3638     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3639     $ret =~ s/^\s+//;
3640
3641     return 1 if $ret eq "";
3642
3643     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3644     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3645
3646     die "deleting drive $deviceid failed : $ret\n";
3647 }
3648
3649 sub qemu_deviceaddverify {
3650     my ($vmid, $deviceid) = @_;
3651
3652     for (my $i = 0; $i <= 5; $i++) {
3653          my $devices_list = vm_devices_list($vmid);
3654          return 1 if defined($devices_list->{$deviceid});
3655          sleep 1;
3656     }
3657
3658     die "error on hotplug device '$deviceid'\n";
3659 }
3660
3661
3662 sub qemu_devicedelverify {
3663     my ($vmid, $deviceid) = @_;
3664
3665     # need to verify that the device is correctly removed as device_del
3666     # is async and empty return is not reliable
3667
3668     for (my $i = 0; $i <= 5; $i++) {
3669          my $devices_list = vm_devices_list($vmid);
3670          return 1 if !defined($devices_list->{$deviceid});
3671          sleep 1;
3672     }
3673
3674     die "error on hot-unplugging device '$deviceid'\n";
3675 }
3676
3677 sub qemu_findorcreatescsihw {
3678     my ($storecfg, $conf, $vmid, $device) = @_;
3679
3680     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3681
3682     my $scsihwid="$controller_prefix$controller";
3683     my $devices_list = vm_devices_list($vmid);
3684
3685     if(!defined($devices_list->{$scsihwid})) {
3686         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3687     }
3688
3689     return 1;
3690 }
3691
3692 sub qemu_deletescsihw {
3693     my ($conf, $vmid, $opt) = @_;
3694
3695     my $device = parse_drive($opt, $conf->{$opt});
3696
3697     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3698         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3699         return 1;
3700     }
3701
3702     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3703
3704     my $devices_list = vm_devices_list($vmid);
3705     foreach my $opt (keys %{$devices_list}) {
3706         if (PVE::QemuServer::is_valid_drivename($opt)) {
3707             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3708             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3709                 return 1;
3710             }
3711         }
3712     }
3713
3714     my $scsihwid="scsihw$controller";
3715
3716     vm_deviceunplug($vmid, $conf, $scsihwid);
3717
3718     return 1;
3719 }
3720
3721 sub qemu_add_pci_bridge {
3722     my ($storecfg, $conf, $vmid, $device) = @_;
3723
3724     my $bridges = {};
3725
3726     my $bridgeid;
3727
3728     print_pci_addr($device, $bridges);
3729
3730     while (my ($k, $v) = each %$bridges) {
3731         $bridgeid = $k;
3732     }
3733     return 1 if !defined($bridgeid) || $bridgeid < 1;
3734
3735     my $bridge = "pci.$bridgeid";
3736     my $devices_list = vm_devices_list($vmid);
3737
3738     if (!defined($devices_list->{$bridge})) {
3739         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3740     }
3741
3742     return 1;
3743 }
3744
3745 sub qemu_set_link_status {
3746     my ($vmid, $device, $up) = @_;
3747
3748     vm_mon_cmd($vmid, "set_link", name => $device,
3749                up => $up ? JSON::true : JSON::false);
3750 }
3751
3752 sub qemu_netdevadd {
3753     my ($vmid, $conf, $device, $deviceid) = @_;
3754
3755     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3756     my %options =  split(/[=,]/, $netdev);
3757
3758     vm_mon_cmd($vmid, "netdev_add",  %options);
3759     return 1;
3760 }
3761
3762 sub qemu_netdevdel {
3763     my ($vmid, $deviceid) = @_;
3764
3765     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3766 }
3767
3768 sub qemu_cpu_hotplug {
3769     my ($vmid, $conf, $vcpus) = @_;
3770
3771     my $sockets = 1;
3772     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3773     $sockets = $conf->{sockets} if  $conf->{sockets};
3774     my $cores = $conf->{cores} || 1;
3775     my $maxcpus = $sockets * $cores;
3776
3777     $vcpus = $maxcpus if !$vcpus;
3778
3779     die "you can't add more vcpus than maxcpus\n"
3780         if $vcpus > $maxcpus;
3781
3782     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3783     die "online cpu unplug is not yet possible\n"
3784         if $vcpus < $currentvcpus;
3785
3786     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3787     die "vcpus in running vm is different than configuration\n"
3788         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3789
3790     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3791         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3792     }
3793 }
3794
3795 sub qemu_memory_hotplug {
3796     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3797
3798     return $value if !check_running($vmid);
3799
3800     my $memory = $conf->{memory} || $defaults->{memory};
3801     $value = $defaults->{memory} if !$value;
3802     return $value if $value == $memory;
3803
3804     my $static_memory = $STATICMEM;
3805     my $dimm_memory = $memory - $static_memory;
3806
3807     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3808     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3809
3810
3811     my $sockets = 1;
3812     $sockets = $conf->{sockets} if $conf->{sockets};
3813
3814     if($value > $memory) {
3815
3816         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3817             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3818
3819                 return if $current_size <= $conf->{memory};
3820
3821                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3822                 if (my $err = $@) {
3823                     eval { qemu_objectdel($vmid, "mem-$name"); };
3824                     die $err;
3825                 }
3826
3827                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3828                 if (my $err = $@) {
3829                     eval { qemu_objectdel($vmid, "mem-$name"); };
3830                     die $err;
3831                 }
3832                 #update conf after each succesful module hotplug
3833                 $conf->{memory} = $current_size;
3834                 PVE::QemuConfig->write_config($vmid, $conf);
3835         });
3836
3837     } else {
3838
3839         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3840             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3841
3842                 return if $current_size >= $conf->{memory};
3843                 print "try to unplug memory dimm $name\n";
3844
3845                 my $retry = 0;
3846                 while (1) {
3847                     eval { qemu_devicedel($vmid, $name) };
3848                     sleep 3;
3849                     my $dimm_list = qemu_dimm_list($vmid);
3850                     last if !$dimm_list->{$name};
3851                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3852                     $retry++;
3853                 }
3854
3855                 #update conf after each succesful module unplug
3856                 $conf->{memory} = $current_size;
3857
3858                 eval { qemu_objectdel($vmid, "mem-$name"); };
3859                 PVE::QemuConfig->write_config($vmid, $conf);
3860         });
3861     }
3862 }
3863
3864 sub qemu_dimm_list {
3865     my ($vmid) = @_;
3866
3867     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3868     my $dimms = {};
3869
3870     foreach my $dimm (@$dimmarray) {
3871
3872         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3873         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3874         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3875         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3876         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3877     }
3878     return $dimms;
3879 }
3880
3881 sub qemu_block_set_io_throttle {
3882     my ($vmid, $deviceid,
3883         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3884         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3885
3886     return if !check_running($vmid) ;
3887
3888     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3889         bps => int($bps),
3890         bps_rd => int($bps_rd),
3891         bps_wr => int($bps_wr),
3892         iops => int($iops),
3893         iops_rd => int($iops_rd),
3894         iops_wr => int($iops_wr),
3895         bps_max => int($bps_max),
3896         bps_rd_max => int($bps_rd_max),
3897         bps_wr_max => int($bps_wr_max),
3898         iops_max => int($iops_max),
3899         iops_rd_max => int($iops_rd_max),
3900         iops_wr_max => int($iops_wr_max)
3901     );
3902
3903 }
3904
3905 # old code, only used to shutdown old VM after update
3906 sub __read_avail {
3907     my ($fh, $timeout) = @_;
3908
3909     my $sel = new IO::Select;
3910     $sel->add($fh);
3911
3912     my $res = '';
3913     my $buf;
3914
3915     my @ready;
3916     while (scalar (@ready = $sel->can_read($timeout))) {
3917         my $count;
3918         if ($count = $fh->sysread($buf, 8192)) {
3919             if ($buf =~ /^(.*)\(qemu\) $/s) {
3920                 $res .= $1;
3921                 last;
3922             } else {
3923                 $res .= $buf;
3924             }
3925         } else {
3926             if (!defined($count)) {
3927                 die "$!\n";
3928             }
3929             last;
3930         }
3931     }
3932
3933     die "monitor read timeout\n" if !scalar(@ready);
3934
3935     return $res;
3936 }
3937
3938 # old code, only used to shutdown old VM after update
3939 sub vm_monitor_command {
3940     my ($vmid, $cmdstr, $nocheck) = @_;
3941
3942     my $res;
3943
3944     eval {
3945         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3946
3947         my $sname = "${var_run_tmpdir}/$vmid.mon";
3948
3949         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3950             die "unable to connect to VM $vmid socket - $!\n";
3951
3952         my $timeout = 3;
3953
3954         # hack: migrate sometime blocks the monitor (when migrate_downtime
3955         # is set)
3956         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3957             $timeout = 60*60; # 1 hour
3958         }
3959
3960         # read banner;
3961         my $data = __read_avail($sock, $timeout);
3962
3963         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3964             die "got unexpected qemu monitor banner\n";
3965         }
3966
3967         my $sel = new IO::Select;
3968         $sel->add($sock);
3969
3970         if (!scalar(my @ready = $sel->can_write($timeout))) {
3971             die "monitor write error - timeout";
3972         }
3973
3974         my $fullcmd = "$cmdstr\r";
3975
3976         # syslog('info', "VM $vmid monitor command: $cmdstr");
3977
3978         my $b;
3979         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3980             die "monitor write error - $!";
3981         }
3982
3983         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3984
3985         $timeout = 20;
3986
3987         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3988             $timeout = 60*60; # 1 hour
3989         } elsif ($cmdstr =~ m/^(eject|change)/) {
3990             $timeout = 60; # note: cdrom mount command is slow
3991         }
3992         if ($res = __read_avail($sock, $timeout)) {
3993
3994             my @lines = split("\r?\n", $res);
3995
3996             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3997
3998             $res = join("\n", @lines);
3999             $res .= "\n";
4000         }
4001     };
4002
4003     my $err = $@;
4004
4005     if ($err) {
4006         syslog("err", "VM $vmid monitor command failed - $err");
4007         die $err;
4008     }
4009
4010     return $res;
4011 }
4012
4013 sub qemu_block_resize {
4014     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4015
4016     my $running = check_running($vmid);
4017
4018     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4019
4020     return if !$running;
4021
4022     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4023
4024 }
4025
4026 sub qemu_volume_snapshot {
4027     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4028
4029     my $running = check_running($vmid);
4030
4031     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4032         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4033     } else {
4034         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4035     }
4036 }
4037
4038 sub qemu_volume_snapshot_delete {
4039     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4040
4041     my $running = check_running($vmid);
4042
4043     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4044
4045     return if !$running;
4046
4047     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4048 }
4049
4050 sub set_migration_caps {
4051     my ($vmid) = @_;
4052
4053     my $cap_ref = [];
4054
4055     my $enabled_cap = {
4056         "auto-converge" => 1,
4057         "xbzrle" => 1,
4058         "x-rdma-pin-all" => 0,
4059         "zero-blocks" => 0,
4060         "compress" => 0
4061     };
4062
4063     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4064
4065     for my $supported_capability (@$supported_capabilities) {
4066         push @$cap_ref, {
4067             capability => $supported_capability->{capability},
4068             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4069         };
4070     }
4071
4072     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4073 }
4074
4075 my $fast_plug_option = {
4076     'lock' => 1,
4077     'name' => 1,
4078     'onboot' => 1,
4079     'shares' => 1,
4080     'startup' => 1,
4081     'description' => 1,
4082 };
4083
4084 # hotplug changes in [PENDING]
4085 # $selection hash can be used to only apply specified options, for
4086 # example: { cores => 1 } (only apply changed 'cores')
4087 # $errors ref is used to return error messages
4088 sub vmconfig_hotplug_pending {
4089     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4090
4091     my $defaults = load_defaults();
4092
4093     # commit values which do not have any impact on running VM first
4094     # Note: those option cannot raise errors, we we do not care about
4095     # $selection and always apply them.
4096
4097     my $add_error = sub {
4098         my ($opt, $msg) = @_;
4099         $errors->{$opt} = "hotplug problem - $msg";
4100     };
4101
4102     my $changes = 0;
4103     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4104         if ($fast_plug_option->{$opt}) {
4105             $conf->{$opt} = $conf->{pending}->{$opt};
4106             delete $conf->{pending}->{$opt};
4107             $changes = 1;
4108         }
4109     }
4110
4111     if ($changes) {
4112         PVE::QemuConfig->write_config($vmid, $conf);
4113         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4114     }
4115
4116     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4117
4118     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4119     while (my ($opt, $force) = each %$pending_delete_hash) {
4120         next if $selection && !$selection->{$opt};
4121         eval {
4122             if ($opt eq 'hotplug') {
4123                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4124             } elsif ($opt eq 'tablet') {
4125                 die "skip\n" if !$hotplug_features->{usb};
4126                 if ($defaults->{tablet}) {
4127                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4128                 } else {
4129                     vm_deviceunplug($vmid, $conf, $opt);
4130                 }
4131             } elsif ($opt eq 'vcpus') {
4132                 die "skip\n" if !$hotplug_features->{cpu};
4133                 qemu_cpu_hotplug($vmid, $conf, undef);
4134             } elsif ($opt eq 'balloon') {
4135                 # enable balloon device is not hotpluggable
4136                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4137             } elsif ($fast_plug_option->{$opt}) {
4138                 # do nothing
4139             } elsif ($opt =~ m/^net(\d+)$/) {
4140                 die "skip\n" if !$hotplug_features->{network};
4141                 vm_deviceunplug($vmid, $conf, $opt);
4142             } elsif (is_valid_drivename($opt)) {
4143                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4144                 vm_deviceunplug($vmid, $conf, $opt);
4145                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4146             } elsif ($opt =~ m/^memory$/) {
4147                 die "skip\n" if !$hotplug_features->{memory};
4148                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4149             } elsif ($opt eq 'cpuunits') {
4150                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4151             } elsif ($opt eq 'cpulimit') {
4152                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4153             } else {
4154                 die "skip\n";
4155             }
4156         };
4157         if (my $err = $@) {
4158             &$add_error($opt, $err) if $err ne "skip\n";
4159         } else {
4160             # save new config if hotplug was successful
4161             delete $conf->{$opt};
4162             vmconfig_undelete_pending_option($conf, $opt);
4163             PVE::QemuConfig->write_config($vmid, $conf);
4164             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4165         }
4166     }
4167
4168     foreach my $opt (keys %{$conf->{pending}}) {
4169         next if $selection && !$selection->{$opt};
4170         my $value = $conf->{pending}->{$opt};
4171         eval {
4172             if ($opt eq 'hotplug') {
4173                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4174             } elsif ($opt eq 'tablet') {
4175                 die "skip\n" if !$hotplug_features->{usb};
4176                 if ($value == 1) {
4177                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4178                 } elsif ($value == 0) {
4179                     vm_deviceunplug($vmid, $conf, $opt);
4180                 }
4181             } elsif ($opt eq 'vcpus') {
4182                 die "skip\n" if !$hotplug_features->{cpu};
4183                 qemu_cpu_hotplug($vmid, $conf, $value);
4184             } elsif ($opt eq 'balloon') {
4185                 # enable/disable balloning device is not hotpluggable
4186                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4187                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4188                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4189
4190                 # allow manual ballooning if shares is set to zero
4191                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4192                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4193                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4194                 }
4195             } elsif ($opt =~ m/^net(\d+)$/) {
4196                 # some changes can be done without hotplug
4197                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4198                                     $vmid, $opt, $value);
4199             } elsif (is_valid_drivename($opt)) {
4200                 # some changes can be done without hotplug
4201                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4202                                      $vmid, $opt, $value, 1);
4203             } elsif ($opt =~ m/^memory$/) { #dimms
4204                 die "skip\n" if !$hotplug_features->{memory};
4205                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4206             } elsif ($opt eq 'cpuunits') {
4207                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4208             } elsif ($opt eq 'cpulimit') {
4209                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4210                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4211             } else {
4212                 die "skip\n";  # skip non-hot-pluggable options
4213             }
4214         };
4215         if (my $err = $@) {
4216             &$add_error($opt, $err) if $err ne "skip\n";
4217         } else {
4218             # save new config if hotplug was successful
4219             $conf->{$opt} = $value;
4220             delete $conf->{pending}->{$opt};
4221             PVE::QemuConfig->write_config($vmid, $conf);
4222             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4223         }
4224     }
4225 }
4226
4227 sub try_deallocate_drive {
4228     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4229
4230     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4231         my $volid = $drive->{file};
4232         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4233             my $sid = PVE::Storage::parse_volume_id($volid);
4234             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4235
4236             # check if the disk is really unused
4237             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4238                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4239             PVE::Storage::vdisk_free($storecfg, $volid);
4240             return 1;
4241         } else {
4242             # If vm is not owner of this disk remove from config
4243             return 1;
4244         }
4245     }
4246
4247     return undef;
4248 }
4249
4250 sub vmconfig_delete_or_detach_drive {
4251     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4252
4253     my $drive = parse_drive($opt, $conf->{$opt});
4254
4255     my $rpcenv = PVE::RPCEnvironment::get();
4256     my $authuser = $rpcenv->get_user();
4257
4258     if ($force) {
4259         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4260         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4261     } else {
4262         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4263     }
4264 }
4265
4266 sub vmconfig_apply_pending {
4267     my ($vmid, $conf, $storecfg) = @_;
4268
4269     # cold plug
4270
4271     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4272     while (my ($opt, $force) = each %$pending_delete_hash) {
4273         die "internal error" if $opt =~ m/^unused/;
4274         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4275         if (!defined($conf->{$opt})) {
4276             vmconfig_undelete_pending_option($conf, $opt);
4277             PVE::QemuConfig->write_config($vmid, $conf);
4278         } elsif (is_valid_drivename($opt)) {
4279             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4280             vmconfig_undelete_pending_option($conf, $opt);
4281             delete $conf->{$opt};
4282             PVE::QemuConfig->write_config($vmid, $conf);
4283         } else {
4284             vmconfig_undelete_pending_option($conf, $opt);
4285             delete $conf->{$opt};
4286             PVE::QemuConfig->write_config($vmid, $conf);
4287         }
4288     }
4289
4290     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4291
4292     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4293         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4294
4295         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4296             # skip if nothing changed
4297         } elsif (is_valid_drivename($opt)) {
4298             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4299                 if defined($conf->{$opt});
4300             $conf->{$opt} = $conf->{pending}->{$opt};
4301         } else {
4302             $conf->{$opt} = $conf->{pending}->{$opt};
4303         }
4304
4305         delete $conf->{pending}->{$opt};
4306         PVE::QemuConfig->write_config($vmid, $conf);
4307     }
4308 }
4309
4310 my $safe_num_ne = sub {
4311     my ($a, $b) = @_;
4312
4313     return 0 if !defined($a) && !defined($b);
4314     return 1 if !defined($a);
4315     return 1 if !defined($b);
4316
4317     return $a != $b;
4318 };
4319
4320 my $safe_string_ne = sub {
4321     my ($a, $b) = @_;
4322
4323     return 0 if !defined($a) && !defined($b);
4324     return 1 if !defined($a);
4325     return 1 if !defined($b);
4326
4327     return $a ne $b;
4328 };
4329
4330 sub vmconfig_update_net {
4331     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4332
4333     my $newnet = parse_net($value);
4334
4335     if ($conf->{$opt}) {
4336         my $oldnet = parse_net($conf->{$opt});
4337
4338         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4339             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4340             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4341             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4342
4343             # for non online change, we try to hot-unplug
4344             die "skip\n" if !$hotplug;
4345             vm_deviceunplug($vmid, $conf, $opt);
4346         } else {
4347
4348             die "internal error" if $opt !~ m/net(\d+)/;
4349             my $iface = "tap${vmid}i$1";
4350
4351             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4352                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4353                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4354                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4355                 PVE::Network::tap_unplug($iface);
4356                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4357             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4358                 # Rate can be applied on its own but any change above needs to
4359                 # include the rate in tap_plug since OVS resets everything.
4360                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4361             }
4362
4363             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4364                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4365             }
4366
4367             return 1;
4368         }
4369     }
4370
4371     if ($hotplug) {
4372         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4373     } else {
4374         die "skip\n";
4375     }
4376 }
4377
4378 sub vmconfig_update_disk {
4379     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4380
4381     # fixme: do we need force?
4382
4383     my $drive = parse_drive($opt, $value);
4384
4385     if ($conf->{$opt}) {
4386
4387         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4388
4389             my $media = $drive->{media} || 'disk';
4390             my $oldmedia = $old_drive->{media} || 'disk';
4391             die "unable to change media type\n" if $media ne $oldmedia;
4392
4393             if (!drive_is_cdrom($old_drive)) {
4394
4395                 if ($drive->{file} ne $old_drive->{file}) {
4396
4397                     die "skip\n" if !$hotplug;
4398
4399                     # unplug and register as unused
4400                     vm_deviceunplug($vmid, $conf, $opt);
4401                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4402
4403                 } else {
4404                     # update existing disk
4405
4406                     # skip non hotpluggable value
4407                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4408                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4409                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4410                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4411                         die "skip\n";
4412                     }
4413
4414                     # apply throttle
4415                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4416                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4417                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4418                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4419                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4420                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4421                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4422                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4423                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4424                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4425                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4426                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4427
4428                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4429                                                    ($drive->{mbps} || 0)*1024*1024,
4430                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4431                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4432                                                    $drive->{iops} || 0,
4433                                                    $drive->{iops_rd} || 0,
4434                                                    $drive->{iops_wr} || 0,
4435                                                    ($drive->{mbps_max} || 0)*1024*1024,
4436                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4437                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4438                                                    $drive->{iops_max} || 0,
4439                                                    $drive->{iops_rd_max} || 0,
4440                                                    $drive->{iops_wr_max} || 0);
4441
4442                     }
4443
4444                     return 1;
4445                 }
4446
4447             } else { # cdrom
4448
4449                 if ($drive->{file} eq 'none') {
4450                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4451                 } else {
4452                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4453                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4454                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4455                 }
4456
4457                 return 1;
4458             }
4459         }
4460     }
4461
4462     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4463     # hotplug new disks
4464     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4465     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4466 }
4467
4468 sub vm_start {
4469     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4470         $forcemachine, $spice_ticket) = @_;
4471
4472     PVE::QemuConfig->lock_config($vmid, sub {
4473         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4474
4475         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4476
4477         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4478
4479         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4480
4481         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4482             vmconfig_apply_pending($vmid, $conf, $storecfg);
4483             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4484         }
4485
4486         my $defaults = load_defaults();
4487
4488         # set environment variable useful inside network script
4489         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4490
4491         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4492
4493         my $migrate_port = 0;
4494         my $migrate_uri;
4495         if ($statefile) {
4496             if ($statefile eq 'tcp') {
4497                 my $localip = "localhost";
4498                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4499                 my $nodename = PVE::INotify::nodename();
4500                 if ($datacenterconf->{migration_unsecure}) {
4501                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4502                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4503                 }
4504                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4505                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4506                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4507                 push @$cmd, '-incoming', $migrate_uri;
4508                 push @$cmd, '-S';
4509             } else {
4510                 push @$cmd, '-loadstate', $statefile;
4511             }
4512         } elsif ($paused) {
4513             push @$cmd, '-S';
4514         }
4515
4516         # host pci devices
4517         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4518           my $d = parse_hostpci($conf->{"hostpci$i"});
4519           next if !$d;
4520           my $pcidevices = $d->{pciid};
4521           foreach my $pcidevice (@$pcidevices) {
4522                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4523
4524                 my $info = pci_device_info("0000:$pciid");
4525                 die "IOMMU not present\n" if !check_iommu_support();
4526                 die "no pci device info for device '$pciid'\n" if !$info;
4527                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4528                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4529           }
4530         }
4531
4532         PVE::Storage::activate_volumes($storecfg, $vollist);
4533
4534         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4535                     umask => 0077); };
4536
4537         if (my $err = $@) {
4538             # deactivate volumes if start fails
4539             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4540             die "start failed: $err";
4541         }
4542
4543         print "migration listens on $migrate_uri\n" if $migrate_uri;
4544
4545         if ($statefile && $statefile ne 'tcp')  {
4546             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4547             warn $@ if $@;
4548         }
4549
4550         if ($migratedfrom) {
4551
4552             eval {
4553                 set_migration_caps($vmid);
4554             };
4555             warn $@ if $@;
4556
4557             if ($spice_port) {
4558                 print "spice listens on port $spice_port\n";
4559                 if ($spice_ticket) {
4560                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4561                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4562                 }
4563             }
4564
4565         } else {
4566
4567             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4568                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4569                     if $conf->{balloon};
4570             }
4571
4572             foreach my $opt (keys %$conf) {
4573                 next if $opt !~  m/^net\d+$/;
4574                 my $nicconf = parse_net($conf->{$opt});
4575                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4576             }
4577         }
4578
4579         vm_mon_cmd_nocheck($vmid, 'qom-set',
4580                     path => "machine/peripheral/balloon0",
4581                     property => "guest-stats-polling-interval",
4582                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4583
4584     });
4585 }
4586
4587 sub vm_mon_cmd {
4588     my ($vmid, $execute, %params) = @_;
4589
4590     my $cmd = { execute => $execute, arguments => \%params };
4591     vm_qmp_command($vmid, $cmd);
4592 }
4593
4594 sub vm_mon_cmd_nocheck {
4595     my ($vmid, $execute, %params) = @_;
4596
4597     my $cmd = { execute => $execute, arguments => \%params };
4598     vm_qmp_command($vmid, $cmd, 1);
4599 }
4600
4601 sub vm_qmp_command {
4602     my ($vmid, $cmd, $nocheck) = @_;
4603
4604     my $res;
4605
4606     my $timeout;
4607     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4608         $timeout = $cmd->{arguments}->{timeout};
4609         delete $cmd->{arguments}->{timeout};
4610     }
4611
4612     eval {
4613         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4614         my $sname = qmp_socket($vmid);
4615         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4616             my $qmpclient = PVE::QMPClient->new();
4617
4618             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4619         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4620             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4621                 if scalar(%{$cmd->{arguments}});
4622             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4623         } else {
4624             die "unable to open monitor socket\n";
4625         }
4626     };
4627     if (my $err = $@) {
4628         syslog("err", "VM $vmid qmp command failed - $err");
4629         die $err;
4630     }
4631
4632     return $res;
4633 }
4634
4635 sub vm_human_monitor_command {
4636     my ($vmid, $cmdline) = @_;
4637
4638     my $res;
4639
4640     my $cmd = {
4641         execute => 'human-monitor-command',
4642         arguments => { 'command-line' => $cmdline},
4643     };
4644
4645     return vm_qmp_command($vmid, $cmd);
4646 }
4647
4648 sub vm_commandline {
4649     my ($storecfg, $vmid) = @_;
4650
4651     my $conf = PVE::QemuConfig->load_config($vmid);
4652
4653     my $defaults = load_defaults();
4654
4655     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4656
4657     return join(' ', @$cmd);
4658 }
4659
4660 sub vm_reset {
4661     my ($vmid, $skiplock) = @_;
4662
4663     PVE::QemuConfig->lock_config($vmid, sub {
4664
4665         my $conf = PVE::QemuConfig->load_config($vmid);
4666
4667         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4668
4669         vm_mon_cmd($vmid, "system_reset");
4670     });
4671 }
4672
4673 sub get_vm_volumes {
4674     my ($conf) = @_;
4675
4676     my $vollist = [];
4677     foreach_volid($conf, sub {
4678         my ($volid, $is_cdrom) = @_;
4679
4680         return if $volid =~ m|^/|;
4681
4682         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4683         return if !$sid;
4684
4685         push @$vollist, $volid;
4686     });
4687
4688     return $vollist;
4689 }
4690
4691 sub vm_stop_cleanup {
4692     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4693
4694     eval {
4695
4696         if (!$keepActive) {
4697             my $vollist = get_vm_volumes($conf);
4698             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4699         }
4700
4701         foreach my $ext (qw(mon qmp pid vnc qga)) {
4702             unlink "/var/run/qemu-server/${vmid}.$ext";
4703         }
4704
4705         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4706     };
4707     warn $@ if $@; # avoid errors - just warn
4708 }
4709
4710 # Note: use $nockeck to skip tests if VM configuration file exists.
4711 # We need that when migration VMs to other nodes (files already moved)
4712 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4713 sub vm_stop {
4714     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4715
4716     $force = 1 if !defined($force) && !$shutdown;
4717
4718     if ($migratedfrom){
4719         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4720         kill 15, $pid if $pid;
4721         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4722         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4723         return;
4724     }
4725
4726     PVE::QemuConfig->lock_config($vmid, sub {
4727
4728         my $pid = check_running($vmid, $nocheck);
4729         return if !$pid;
4730
4731         my $conf;
4732         if (!$nocheck) {
4733             $conf = PVE::QemuConfig->load_config($vmid);
4734             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4735             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4736                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4737                 $timeout = $opts->{down} if $opts->{down};
4738             }
4739         }
4740
4741         $timeout = 60 if !defined($timeout);
4742
4743         eval {
4744             if ($shutdown) {
4745                 if (defined($conf) && $conf->{agent}) {
4746                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4747                 } else {
4748                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4749                 }
4750             } else {
4751                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4752             }
4753         };
4754         my $err = $@;
4755
4756         if (!$err) {
4757             my $count = 0;
4758             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4759                 $count++;
4760                 sleep 1;
4761             }
4762
4763             if ($count >= $timeout) {
4764                 if ($force) {
4765                     warn "VM still running - terminating now with SIGTERM\n";
4766                     kill 15, $pid;
4767                 } else {
4768                     die "VM quit/powerdown failed - got timeout\n";
4769                 }
4770             } else {
4771                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4772                 return;
4773             }
4774         } else {
4775             if ($force) {
4776                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4777                 kill 15, $pid;
4778             } else {
4779                 die "VM quit/powerdown failed\n";
4780             }
4781         }
4782
4783         # wait again
4784         $timeout = 10;
4785
4786         my $count = 0;
4787         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4788             $count++;
4789             sleep 1;
4790         }
4791
4792         if ($count >= $timeout) {
4793             warn "VM still running - terminating now with SIGKILL\n";
4794             kill 9, $pid;
4795             sleep 1;
4796         }
4797
4798         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4799    });
4800 }
4801
4802 sub vm_suspend {
4803     my ($vmid, $skiplock) = @_;
4804
4805     PVE::QemuConfig->lock_config($vmid, sub {
4806
4807         my $conf = PVE::QemuConfig->load_config($vmid);
4808
4809         PVE::QemuConfig->check_lock($conf)
4810             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4811
4812         vm_mon_cmd($vmid, "stop");
4813     });
4814 }
4815
4816 sub vm_resume {
4817     my ($vmid, $skiplock, $nocheck) = @_;
4818
4819     PVE::QemuConfig->lock_config($vmid, sub {
4820
4821         if (!$nocheck) {
4822
4823             my $conf = PVE::QemuConfig->load_config($vmid);
4824
4825             PVE::QemuConfig->check_lock($conf)
4826                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4827
4828             vm_mon_cmd($vmid, "cont");
4829
4830         } else {
4831             vm_mon_cmd_nocheck($vmid, "cont");
4832         }
4833     });
4834 }
4835
4836 sub vm_sendkey {
4837     my ($vmid, $skiplock, $key) = @_;
4838
4839     PVE::QemuConfig->lock_config($vmid, sub {
4840
4841         my $conf = PVE::QemuConfig->load_config($vmid);
4842
4843         # there is no qmp command, so we use the human monitor command
4844         vm_human_monitor_command($vmid, "sendkey $key");
4845     });
4846 }
4847
4848 sub vm_destroy {
4849     my ($storecfg, $vmid, $skiplock) = @_;
4850
4851     PVE::QemuConfig->lock_config($vmid, sub {
4852
4853         my $conf = PVE::QemuConfig->load_config($vmid);
4854
4855         if (!check_running($vmid)) {
4856             destroy_vm($storecfg, $vmid, undef, $skiplock);
4857         } else {
4858             die "VM $vmid is running - destroy failed\n";
4859         }
4860     });
4861 }
4862
4863 # pci helpers
4864
4865 sub file_write {
4866     my ($filename, $buf) = @_;
4867
4868     my $fh = IO::File->new($filename, "w");
4869     return undef if !$fh;
4870
4871     my $res = print $fh $buf;
4872
4873     $fh->close();
4874
4875     return $res;
4876 }
4877
4878 sub pci_device_info {
4879     my ($name) = @_;
4880
4881     my $res;
4882
4883     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4884     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4885
4886     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4887     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4888
4889     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4890     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4891
4892     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4893     return undef if !defined($product) || $product !~ s/^0x//;
4894
4895     $res = {
4896         name => $name,
4897         vendor => $vendor,
4898         product => $product,
4899         domain => $domain,
4900         bus => $bus,
4901         slot => $slot,
4902         func => $func,
4903         irq => $irq,
4904         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4905     };
4906
4907     return $res;
4908 }
4909
4910 sub pci_dev_reset {
4911     my ($dev) = @_;
4912
4913     my $name = $dev->{name};
4914
4915     my $fn = "$pcisysfs/devices/$name/reset";
4916
4917     return file_write($fn, "1");
4918 }
4919
4920 sub pci_dev_bind_to_vfio {
4921     my ($dev) = @_;
4922
4923     my $name = $dev->{name};
4924
4925     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4926
4927     if (!-d $vfio_basedir) {
4928         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4929     }
4930     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4931
4932     my $testdir = "$vfio_basedir/$name";
4933     return 1 if -d $testdir;
4934
4935     my $data = "$dev->{vendor} $dev->{product}";
4936     return undef if !file_write("$vfio_basedir/new_id", $data);
4937
4938     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4939     if (!file_write($fn, $name)) {
4940         return undef if -f $fn;
4941     }
4942
4943     $fn = "$vfio_basedir/bind";
4944     if (! -d $testdir) {
4945         return undef if !file_write($fn, $name);
4946     }
4947
4948     return -d $testdir;
4949 }
4950
4951 sub pci_dev_group_bind_to_vfio {
4952     my ($pciid) = @_;
4953
4954     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4955
4956     if (!-d $vfio_basedir) {
4957         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4958     }
4959     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4960
4961     # get IOMMU group devices
4962     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4963       my @devs = grep /^0000:/, readdir($D);
4964     closedir($D);
4965
4966     foreach my $pciid (@devs) {
4967         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4968
4969         # pci bridges, switches or root ports are not supported
4970         # they have a pci_bus subdirectory so skip them
4971         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4972
4973         my $info = pci_device_info($1);
4974         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4975     }
4976
4977     return 1;
4978 }
4979
4980 sub print_pci_addr {
4981     my ($id, $bridges) = @_;
4982
4983     my $res = '';
4984     my $devices = {
4985         piix3 => { bus => 0, addr => 1 },
4986         #addr2 : first videocard
4987         balloon0 => { bus => 0, addr => 3 },
4988         watchdog => { bus => 0, addr => 4 },
4989         scsihw0 => { bus => 0, addr => 5 },
4990         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4991         scsihw1 => { bus => 0, addr => 6 },
4992         ahci0 => { bus => 0, addr => 7 },
4993         qga0 => { bus => 0, addr => 8 },
4994         spice => { bus => 0, addr => 9 },
4995         virtio0 => { bus => 0, addr => 10 },
4996         virtio1 => { bus => 0, addr => 11 },
4997         virtio2 => { bus => 0, addr => 12 },
4998         virtio3 => { bus => 0, addr => 13 },
4999         virtio4 => { bus => 0, addr => 14 },
5000         virtio5 => { bus => 0, addr => 15 },
5001         hostpci0 => { bus => 0, addr => 16 },
5002         hostpci1 => { bus => 0, addr => 17 },
5003         net0 => { bus => 0, addr => 18 },
5004         net1 => { bus => 0, addr => 19 },
5005         net2 => { bus => 0, addr => 20 },
5006         net3 => { bus => 0, addr => 21 },
5007         net4 => { bus => 0, addr => 22 },
5008         net5 => { bus => 0, addr => 23 },
5009         vga1 => { bus => 0, addr => 24 },
5010         vga2 => { bus => 0, addr => 25 },
5011         vga3 => { bus => 0, addr => 26 },
5012         hostpci2 => { bus => 0, addr => 27 },
5013         hostpci3 => { bus => 0, addr => 28 },
5014         #addr29 : usb-host (pve-usb.cfg)
5015         'pci.1' => { bus => 0, addr => 30 },
5016         'pci.2' => { bus => 0, addr => 31 },
5017         'net6' => { bus => 1, addr => 1 },
5018         'net7' => { bus => 1, addr => 2 },
5019         'net8' => { bus => 1, addr => 3 },
5020         'net9' => { bus => 1, addr => 4 },
5021         'net10' => { bus => 1, addr => 5 },
5022         'net11' => { bus => 1, addr => 6 },
5023         'net12' => { bus => 1, addr => 7 },
5024         'net13' => { bus => 1, addr => 8 },
5025         'net14' => { bus => 1, addr => 9 },
5026         'net15' => { bus => 1, addr => 10 },
5027         'net16' => { bus => 1, addr => 11 },
5028         'net17' => { bus => 1, addr => 12 },
5029         'net18' => { bus => 1, addr => 13 },
5030         'net19' => { bus => 1, addr => 14 },
5031         'net20' => { bus => 1, addr => 15 },
5032         'net21' => { bus => 1, addr => 16 },
5033         'net22' => { bus => 1, addr => 17 },
5034         'net23' => { bus => 1, addr => 18 },
5035         'net24' => { bus => 1, addr => 19 },
5036         'net25' => { bus => 1, addr => 20 },
5037         'net26' => { bus => 1, addr => 21 },
5038         'net27' => { bus => 1, addr => 22 },
5039         'net28' => { bus => 1, addr => 23 },
5040         'net29' => { bus => 1, addr => 24 },
5041         'net30' => { bus => 1, addr => 25 },
5042         'net31' => { bus => 1, addr => 26 },
5043         'xhci' => { bus => 1, addr => 27 },
5044         'virtio6' => { bus => 2, addr => 1 },
5045         'virtio7' => { bus => 2, addr => 2 },
5046         'virtio8' => { bus => 2, addr => 3 },
5047         'virtio9' => { bus => 2, addr => 4 },
5048         'virtio10' => { bus => 2, addr => 5 },
5049         'virtio11' => { bus => 2, addr => 6 },
5050         'virtio12' => { bus => 2, addr => 7 },
5051         'virtio13' => { bus => 2, addr => 8 },
5052         'virtio14' => { bus => 2, addr => 9 },
5053         'virtio15' => { bus => 2, addr => 10 },
5054         'virtioscsi0' => { bus => 3, addr => 1 },
5055         'virtioscsi1' => { bus => 3, addr => 2 },
5056         'virtioscsi2' => { bus => 3, addr => 3 },
5057         'virtioscsi3' => { bus => 3, addr => 4 },
5058         'virtioscsi4' => { bus => 3, addr => 5 },
5059         'virtioscsi5' => { bus => 3, addr => 6 },
5060         'virtioscsi6' => { bus => 3, addr => 7 },
5061         'virtioscsi7' => { bus => 3, addr => 8 },
5062         'virtioscsi8' => { bus => 3, addr => 9 },
5063         'virtioscsi9' => { bus => 3, addr => 10 },
5064         'virtioscsi10' => { bus => 3, addr => 11 },
5065         'virtioscsi11' => { bus => 3, addr => 12 },
5066         'virtioscsi12' => { bus => 3, addr => 13 },
5067         'virtioscsi13' => { bus => 3, addr => 14 },
5068         'virtioscsi14' => { bus => 3, addr => 15 },
5069         'virtioscsi15' => { bus => 3, addr => 16 },
5070         'virtioscsi16' => { bus => 3, addr => 17 },
5071         'virtioscsi17' => { bus => 3, addr => 18 },
5072         'virtioscsi18' => { bus => 3, addr => 19 },
5073         'virtioscsi19' => { bus => 3, addr => 20 },
5074         'virtioscsi20' => { bus => 3, addr => 21 },
5075         'virtioscsi21' => { bus => 3, addr => 22 },
5076         'virtioscsi22' => { bus => 3, addr => 23 },
5077         'virtioscsi23' => { bus => 3, addr => 24 },
5078         'virtioscsi24' => { bus => 3, addr => 25 },
5079         'virtioscsi25' => { bus => 3, addr => 26 },
5080         'virtioscsi26' => { bus => 3, addr => 27 },
5081         'virtioscsi27' => { bus => 3, addr => 28 },
5082         'virtioscsi28' => { bus => 3, addr => 29 },
5083         'virtioscsi29' => { bus => 3, addr => 30 },
5084         'virtioscsi30' => { bus => 3, addr => 31 },
5085
5086     };
5087
5088     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5089            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5090            my $bus = $devices->{$id}->{bus};
5091            $res = ",bus=pci.$bus,addr=$addr";
5092            $bridges->{$bus} = 1 if $bridges;
5093     }
5094     return $res;
5095
5096 }
5097
5098 sub print_pcie_addr {
5099     my ($id) = @_;
5100
5101     my $res = '';
5102     my $devices = {
5103         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5104         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5105         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5106         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5107     };
5108
5109     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5110            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5111            my $bus = $devices->{$id}->{bus};
5112            $res = ",bus=$bus,addr=$addr";
5113     }
5114     return $res;
5115
5116 }
5117
5118 # vzdump restore implementaion
5119
5120 sub tar_archive_read_firstfile {
5121     my $archive = shift;
5122
5123     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5124
5125     # try to detect archive type first
5126     my $pid = open (TMP, "tar tf '$archive'|") ||
5127         die "unable to open file '$archive'\n";
5128     my $firstfile = <TMP>;
5129     kill 15, $pid;
5130     close TMP;
5131
5132     die "ERROR: archive contaions no data\n" if !$firstfile;
5133     chomp $firstfile;
5134
5135     return $firstfile;
5136 }
5137
5138 sub tar_restore_cleanup {
5139     my ($storecfg, $statfile) = @_;
5140
5141     print STDERR "starting cleanup\n";
5142
5143     if (my $fd = IO::File->new($statfile, "r")) {
5144         while (defined(my $line = <$fd>)) {
5145             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5146                 my $volid = $2;
5147                 eval {
5148                     if ($volid =~ m|^/|) {
5149                         unlink $volid || die 'unlink failed\n';
5150                     } else {
5151                         PVE::Storage::vdisk_free($storecfg, $volid);
5152                     }
5153                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5154                 };
5155                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5156             } else {
5157                 print STDERR "unable to parse line in statfile - $line";
5158             }
5159         }
5160         $fd->close();
5161     }
5162 }
5163
5164 sub restore_archive {
5165     my ($archive, $vmid, $user, $opts) = @_;
5166
5167     my $format = $opts->{format};
5168     my $comp;
5169
5170     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5171         $format = 'tar' if !$format;
5172         $comp = 'gzip';
5173     } elsif ($archive =~ m/\.tar$/) {
5174         $format = 'tar' if !$format;
5175     } elsif ($archive =~ m/.tar.lzo$/) {
5176         $format = 'tar' if !$format;
5177         $comp = 'lzop';
5178     } elsif ($archive =~ m/\.vma$/) {
5179         $format = 'vma' if !$format;
5180     } elsif ($archive =~ m/\.vma\.gz$/) {
5181         $format = 'vma' if !$format;
5182         $comp = 'gzip';
5183     } elsif ($archive =~ m/\.vma\.lzo$/) {
5184         $format = 'vma' if !$format;
5185         $comp = 'lzop';
5186     } else {
5187         $format = 'vma' if !$format; # default
5188     }
5189
5190     # try to detect archive format
5191     if ($format eq 'tar') {
5192         return restore_tar_archive($archive, $vmid, $user, $opts);
5193     } else {
5194         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5195     }
5196 }
5197
5198 sub restore_update_config_line {
5199     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5200
5201     return if $line =~ m/^\#qmdump\#/;
5202     return if $line =~ m/^\#vzdump\#/;
5203     return if $line =~ m/^lock:/;
5204     return if $line =~ m/^unused\d+:/;
5205     return if $line =~ m/^parent:/;
5206     return if $line =~ m/^template:/; # restored VM is never a template
5207
5208     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5209         # try to convert old 1.X settings
5210         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5211         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5212             my ($model, $macaddr) = split(/\=/, $devconfig);
5213             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5214             my $net = {
5215                 model => $model,
5216                 bridge => "vmbr$ind",
5217                 macaddr => $macaddr,
5218             };
5219             my $netstr = print_net($net);
5220
5221             print $outfd "net$cookie->{netcount}: $netstr\n";
5222             $cookie->{netcount}++;
5223         }
5224     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5225         my ($id, $netstr) = ($1, $2);
5226         my $net = parse_net($netstr);
5227         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5228         $netstr = print_net($net);
5229         print $outfd "$id: $netstr\n";
5230     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5231         my $virtdev = $1;
5232         my $value = $3;
5233         my $di = parse_drive($virtdev, $value);
5234         if (defined($di->{backup}) && !$di->{backup}) {
5235             print $outfd "#$line";
5236         } elsif ($map->{$virtdev}) {
5237             delete $di->{format}; # format can change on restore
5238             $di->{file} = $map->{$virtdev};
5239             $value = print_drive($vmid, $di);
5240             print $outfd "$virtdev: $value\n";
5241         } else {
5242             print $outfd $line;
5243         }
5244     } else {
5245         print $outfd $line;
5246     }
5247 }
5248
5249 sub scan_volids {
5250     my ($cfg, $vmid) = @_;
5251
5252     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5253
5254     my $volid_hash = {};
5255     foreach my $storeid (keys %$info) {
5256         foreach my $item (@{$info->{$storeid}}) {
5257             next if !($item->{volid} && $item->{size});
5258             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5259             $volid_hash->{$item->{volid}} = $item;
5260         }
5261     }
5262
5263     return $volid_hash;
5264 }
5265
5266 sub is_volume_in_use {
5267     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5268
5269     my $path = PVE::Storage::path($storecfg, $volid);
5270
5271     my $scan_config = sub {
5272         my ($cref, $snapname) = @_;
5273
5274         foreach my $key (keys %$cref) {
5275             my $value = $cref->{$key};
5276             if (is_valid_drivename($key)) {
5277                 next if $skip_drive && $key eq $skip_drive;
5278                 my $drive = parse_drive($key, $value);
5279                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5280                 return 1 if $volid eq $drive->{file};
5281                 if ($drive->{file} =~ m!^/!) {
5282                     return 1 if $drive->{file} eq $path;
5283                 } else {
5284                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5285                     next if !$storeid;
5286                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5287                     next if !$scfg;
5288                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5289                 }
5290             }
5291         }
5292
5293         return 0;
5294     };
5295
5296     return 1 if &$scan_config($conf);
5297
5298     undef $skip_drive;
5299
5300     foreach my $snapname (keys %{$conf->{snapshots}}) {
5301         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5302     }
5303
5304     return 0;
5305 }
5306
5307 sub update_disksize {
5308     my ($vmid, $conf, $volid_hash) = @_;
5309
5310     my $changes;
5311
5312     my $used = {};
5313
5314     # Note: it is allowed to define multiple storages with same path (alias), so
5315     # we need to check both 'volid' and real 'path' (two different volid can point
5316     # to the same path).
5317
5318     my $usedpath = {};
5319
5320     # update size info
5321     foreach my $opt (keys %$conf) {
5322         if (is_valid_drivename($opt)) {
5323             my $drive = parse_drive($opt, $conf->{$opt});
5324             my $volid = $drive->{file};
5325             next if !$volid;
5326
5327             $used->{$volid} = 1;
5328             if ($volid_hash->{$volid} &&
5329                 (my $path = $volid_hash->{$volid}->{path})) {
5330                 $usedpath->{$path} = 1;
5331             }
5332
5333             next if drive_is_cdrom($drive);
5334             next if !$volid_hash->{$volid};
5335
5336             $drive->{size} = $volid_hash->{$volid}->{size};
5337             my $new = print_drive($vmid, $drive);
5338             if ($new ne $conf->{$opt}) {
5339                 $changes = 1;
5340                 $conf->{$opt} = $new;
5341             }
5342         }
5343     }
5344
5345     # remove 'unusedX' entry if volume is used
5346     foreach my $opt (keys %$conf) {
5347         next if $opt !~ m/^unused\d+$/;
5348         my $volid = $conf->{$opt};
5349         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5350         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5351             $changes = 1;
5352             delete $conf->{$opt};
5353         }
5354     }
5355
5356     foreach my $volid (sort keys %$volid_hash) {
5357         next if $volid =~ m/vm-$vmid-state-/;
5358         next if $used->{$volid};
5359         my $path = $volid_hash->{$volid}->{path};
5360         next if !$path; # just to be sure
5361         next if $usedpath->{$path};
5362         $changes = 1;
5363         PVE::QemuConfig->add_unused_volume($conf, $volid);
5364         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5365     }
5366
5367     return $changes;
5368 }
5369
5370 sub rescan {
5371     my ($vmid, $nolock) = @_;
5372
5373     my $cfg = PVE::Storage::config();
5374
5375     my $volid_hash = scan_volids($cfg, $vmid);
5376
5377     my $updatefn =  sub {
5378         my ($vmid) = @_;
5379
5380         my $conf = PVE::QemuConfig->load_config($vmid);
5381
5382         PVE::QemuConfig->check_lock($conf);
5383
5384         my $vm_volids = {};
5385         foreach my $volid (keys %$volid_hash) {
5386             my $info = $volid_hash->{$volid};
5387             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5388         }
5389
5390         my $changes = update_disksize($vmid, $conf, $vm_volids);
5391
5392         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5393     };
5394
5395     if (defined($vmid)) {
5396         if ($nolock) {
5397             &$updatefn($vmid);
5398         } else {
5399             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5400         }
5401     } else {
5402         my $vmlist = config_list();
5403         foreach my $vmid (keys %$vmlist) {
5404             if ($nolock) {
5405                 &$updatefn($vmid);
5406             } else {
5407                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5408             }
5409         }
5410     }
5411 }
5412
5413 sub restore_vma_archive {
5414     my ($archive, $vmid, $user, $opts, $comp) = @_;
5415
5416     my $input = $archive eq '-' ? "<&STDIN" : undef;
5417     my $readfrom = $archive;
5418
5419     my $uncomp = '';
5420     if ($comp) {
5421         $readfrom = '-';
5422         my $qarchive = PVE::Tools::shellquote($archive);
5423         if ($comp eq 'gzip') {
5424             $uncomp = "zcat $qarchive|";
5425         } elsif ($comp eq 'lzop') {
5426             $uncomp = "lzop -d -c $qarchive|";
5427         } else {
5428             die "unknown compression method '$comp'\n";
5429         }
5430
5431     }
5432
5433     my $tmpdir = "/var/tmp/vzdumptmp$$";
5434     rmtree $tmpdir;
5435
5436     # disable interrupts (always do cleanups)
5437     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5438         warn "got interrupt - ignored\n";
5439     };
5440
5441     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5442     POSIX::mkfifo($mapfifo, 0600);
5443     my $fifofh;
5444
5445     my $openfifo = sub {
5446         open($fifofh, '>', $mapfifo) || die $!;
5447     };
5448
5449     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5450
5451     my $oldtimeout;
5452     my $timeout = 5;
5453
5454     my $devinfo = {};
5455
5456     my $rpcenv = PVE::RPCEnvironment::get();
5457
5458     my $conffile = PVE::QemuConfig->config_file($vmid);
5459     my $tmpfn = "$conffile.$$.tmp";
5460
5461     # Note: $oldconf is undef if VM does not exists
5462     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5463     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5464
5465     my $print_devmap = sub {
5466         my $virtdev_hash = {};
5467
5468         my $cfgfn = "$tmpdir/qemu-server.conf";
5469
5470         # we can read the config - that is already extracted
5471         my $fh = IO::File->new($cfgfn, "r") ||
5472             "unable to read qemu-server.conf - $!\n";
5473
5474         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5475         if (-f $fwcfgfn) {
5476             my $pve_firewall_dir = '/etc/pve/firewall';
5477             mkdir $pve_firewall_dir; # make sure the dir exists
5478             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5479         }
5480
5481         while (defined(my $line = <$fh>)) {
5482             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5483                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5484                 die "archive does not contain data for drive '$virtdev'\n"
5485                     if !$devinfo->{$devname};
5486                 if (defined($opts->{storage})) {
5487                     $storeid = $opts->{storage} || 'local';
5488                 } elsif (!$storeid) {
5489                     $storeid = 'local';
5490                 }
5491                 $format = 'raw' if !$format;
5492                 $devinfo->{$devname}->{devname} = $devname;
5493                 $devinfo->{$devname}->{virtdev} = $virtdev;
5494                 $devinfo->{$devname}->{format} = $format;
5495                 $devinfo->{$devname}->{storeid} = $storeid;
5496
5497                 # check permission on storage
5498                 my $pool = $opts->{pool}; # todo: do we need that?
5499                 if ($user ne 'root@pam') {
5500                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5501                 }
5502
5503                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5504             }
5505         }
5506
5507         foreach my $devname (keys %$devinfo) {
5508             die "found no device mapping information for device '$devname'\n"
5509                 if !$devinfo->{$devname}->{virtdev};
5510         }
5511
5512         my $cfg = PVE::Storage::config();
5513
5514         # create empty/temp config
5515         if ($oldconf) {
5516             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5517             foreach_drive($oldconf, sub {
5518                 my ($ds, $drive) = @_;
5519
5520                 return if drive_is_cdrom($drive);
5521
5522                 my $volid = $drive->{file};
5523
5524                 return if !$volid || $volid =~ m|^/|;
5525
5526                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5527                 return if !$path || !$owner || ($owner != $vmid);
5528
5529                 # Note: only delete disk we want to restore
5530                 # other volumes will become unused
5531                 if ($virtdev_hash->{$ds}) {
5532                     PVE::Storage::vdisk_free($cfg, $volid);
5533                 }
5534             });
5535
5536             # delete vmstate files
5537             # since after the restore we have no snapshots anymore
5538             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5539                 my $snap = $oldconf->{snapshots}->{$snapname};
5540                 if ($snap->{vmstate}) {
5541                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5542                     if (my $err = $@) {
5543                         warn $err;
5544                     }
5545                 }
5546             }
5547         }
5548
5549         my $map = {};
5550         foreach my $virtdev (sort keys %$virtdev_hash) {
5551             my $d = $virtdev_hash->{$virtdev};
5552             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5553             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5554
5555             # test if requested format is supported
5556             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5557             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5558             $d->{format} = $defFormat if !$supported;
5559
5560             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5561                                                   $d->{format}, undef, $alloc_size);
5562             print STDERR "new volume ID is '$volid'\n";
5563             $d->{volid} = $volid;
5564             my $path = PVE::Storage::path($cfg, $volid);
5565
5566             PVE::Storage::activate_volumes($cfg,[$volid]);
5567
5568             my $write_zeros = 1;
5569             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5570                 $write_zeros = 0;
5571             }
5572
5573             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5574
5575             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5576             $map->{$virtdev} = $volid;
5577         }
5578
5579         $fh->seek(0, 0) || die "seek failed - $!\n";
5580
5581         my $outfd = new IO::File ($tmpfn, "w") ||
5582             die "unable to write config for VM $vmid\n";
5583
5584         my $cookie = { netcount => 0 };
5585         while (defined(my $line = <$fh>)) {
5586             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5587         }
5588
5589         $fh->close();
5590         $outfd->close();
5591     };
5592
5593     eval {
5594         # enable interrupts
5595         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5596             die "interrupted by signal\n";
5597         };
5598         local $SIG{ALRM} = sub { die "got timeout\n"; };
5599
5600         $oldtimeout = alarm($timeout);
5601
5602         my $parser = sub {
5603             my $line = shift;
5604
5605             print "$line\n";
5606
5607             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5608                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5609                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5610             } elsif ($line =~ m/^CTIME: /) {
5611                 # we correctly received the vma config, so we can disable
5612                 # the timeout now for disk allocation (set to 10 minutes, so
5613                 # that we always timeout if something goes wrong)
5614                 alarm(600);
5615                 &$print_devmap();
5616                 print $fifofh "done\n";
5617                 my $tmp = $oldtimeout || 0;
5618                 $oldtimeout = undef;
5619                 alarm($tmp);
5620                 close($fifofh);
5621             }
5622         };
5623
5624         print "restore vma archive: $cmd\n";
5625         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5626     };
5627     my $err = $@;
5628
5629     alarm($oldtimeout) if $oldtimeout;
5630
5631     my $vollist = [];
5632     foreach my $devname (keys %$devinfo) {
5633         my $volid = $devinfo->{$devname}->{volid};
5634         push @$vollist, $volid if $volid;
5635     }
5636
5637     my $cfg = PVE::Storage::config();
5638     PVE::Storage::deactivate_volumes($cfg, $vollist);
5639
5640     unlink $mapfifo;
5641
5642     if ($err) {
5643         rmtree $tmpdir;
5644         unlink $tmpfn;
5645
5646         foreach my $devname (keys %$devinfo) {
5647             my $volid = $devinfo->{$devname}->{volid};
5648             next if !$volid;
5649             eval {
5650                 if ($volid =~ m|^/|) {
5651                     unlink $volid || die 'unlink failed\n';
5652                 } else {
5653                     PVE::Storage::vdisk_free($cfg, $volid);
5654                 }
5655                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5656             };
5657             print STDERR "unable to cleanup '$volid' - $@" if $@;
5658         }
5659         die $err;
5660     }
5661
5662     rmtree $tmpdir;
5663
5664     rename($tmpfn, $conffile) ||
5665         die "unable to commit configuration file '$conffile'\n";
5666
5667     PVE::Cluster::cfs_update(); # make sure we read new file
5668
5669     eval { rescan($vmid, 1); };
5670     warn $@ if $@;
5671 }
5672
5673 sub restore_tar_archive {
5674     my ($archive, $vmid, $user, $opts) = @_;
5675
5676     if ($archive ne '-') {
5677         my $firstfile = tar_archive_read_firstfile($archive);
5678         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5679             if $firstfile ne 'qemu-server.conf';
5680     }
5681
5682     my $storecfg = PVE::Storage::config();
5683
5684     # destroy existing data - keep empty config
5685     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5686     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5687
5688     my $tocmd = "/usr/lib/qemu-server/qmextract";
5689
5690     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5691     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5692     $tocmd .= ' --prealloc' if $opts->{prealloc};
5693     $tocmd .= ' --info' if $opts->{info};
5694
5695     # tar option "xf" does not autodetect compression when read from STDIN,
5696     # so we pipe to zcat
5697     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5698         PVE::Tools::shellquote("--to-command=$tocmd");
5699
5700     my $tmpdir = "/var/tmp/vzdumptmp$$";
5701     mkpath $tmpdir;
5702
5703     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5704     local $ENV{VZDUMP_VMID} = $vmid;
5705     local $ENV{VZDUMP_USER} = $user;
5706
5707     my $conffile = PVE::QemuConfig->config_file($vmid);
5708     my $tmpfn = "$conffile.$$.tmp";
5709
5710     # disable interrupts (always do cleanups)
5711     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5712         print STDERR "got interrupt - ignored\n";
5713     };
5714
5715     eval {
5716         # enable interrupts
5717         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5718             die "interrupted by signal\n";
5719         };
5720
5721         if ($archive eq '-') {
5722             print "extracting archive from STDIN\n";
5723             run_command($cmd, input => "<&STDIN");
5724         } else {
5725             print "extracting archive '$archive'\n";
5726             run_command($cmd);
5727         }
5728
5729         return if $opts->{info};
5730
5731         # read new mapping
5732         my $map = {};
5733         my $statfile = "$tmpdir/qmrestore.stat";
5734         if (my $fd = IO::File->new($statfile, "r")) {
5735             while (defined (my $line = <$fd>)) {
5736                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5737                     $map->{$1} = $2 if $1;
5738                 } else {
5739                     print STDERR "unable to parse line in statfile - $line\n";
5740                 }
5741             }
5742             $fd->close();
5743         }
5744
5745         my $confsrc = "$tmpdir/qemu-server.conf";
5746
5747         my $srcfd = new IO::File($confsrc, "r") ||
5748             die "unable to open file '$confsrc'\n";
5749
5750         my $outfd = new IO::File ($tmpfn, "w") ||
5751             die "unable to write config for VM $vmid\n";
5752
5753         my $cookie = { netcount => 0 };
5754         while (defined (my $line = <$srcfd>)) {
5755             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5756         }
5757
5758         $srcfd->close();
5759         $outfd->close();
5760     };
5761     my $err = $@;
5762
5763     if ($err) {
5764
5765         unlink $tmpfn;
5766
5767         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5768
5769         die $err;
5770     }
5771
5772     rmtree $tmpdir;
5773
5774     rename $tmpfn, $conffile ||
5775         die "unable to commit configuration file '$conffile'\n";
5776
5777     PVE::Cluster::cfs_update(); # make sure we read new file
5778
5779     eval { rescan($vmid, 1); };
5780     warn $@ if $@;
5781 };
5782
5783 sub foreach_writable_storage {
5784     my ($conf, $func) = @_;
5785
5786     my $sidhash = {};
5787
5788     foreach my $ds (keys %$conf) {
5789         next if !is_valid_drivename($ds);
5790
5791         my $drive = parse_drive($ds, $conf->{$ds});
5792         next if !$drive;
5793         next if drive_is_cdrom($drive);
5794
5795         my $volid = $drive->{file};
5796
5797         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5798         $sidhash->{$sid} = $sid if $sid;
5799     }
5800
5801     foreach my $sid (sort keys %$sidhash) {
5802         &$func($sid);
5803     }
5804 }
5805
5806 sub do_snapshots_with_qemu {
5807     my ($storecfg, $volid) = @_;
5808
5809     my $storage_name = PVE::Storage::parse_volume_id($volid);
5810
5811     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5812         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5813         return 1;
5814     }
5815
5816     if ($volid =~ m/\.(qcow2|qed)$/){
5817         return 1;
5818     }
5819
5820     return undef;
5821 }
5822
5823 sub qga_check_running {
5824     my ($vmid) = @_;
5825
5826     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5827     if ($@) {
5828         warn "Qemu Guest Agent are not running - $@";
5829         return 0;
5830     }
5831     return 1;
5832 }
5833
5834 sub template_create {
5835     my ($vmid, $conf, $disk) = @_;
5836
5837     my $storecfg = PVE::Storage::config();
5838
5839     foreach_drive($conf, sub {
5840         my ($ds, $drive) = @_;
5841
5842         return if drive_is_cdrom($drive);
5843         return if $disk && $ds ne $disk;
5844
5845         my $volid = $drive->{file};
5846         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5847
5848         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5849         $drive->{file} = $voliddst;
5850         $conf->{$ds} = print_drive($vmid, $drive);
5851         PVE::QemuConfig->write_config($vmid, $conf);
5852     });
5853 }
5854
5855 sub qemu_img_convert {
5856     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5857
5858     my $storecfg = PVE::Storage::config();
5859     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5860     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5861
5862     if ($src_storeid && $dst_storeid) {
5863
5864         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5865
5866         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5867         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5868
5869         my $src_format = qemu_img_format($src_scfg, $src_volname);
5870         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5871
5872         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5873         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5874
5875         my $cmd = [];
5876         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5877         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5878         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5879         if ($is_zero_initialized) {
5880             push @$cmd, "zeroinit:$dst_path";
5881         } else {
5882             push @$cmd, $dst_path;
5883         }
5884
5885         my $parser = sub {
5886             my $line = shift;
5887             if($line =~ m/\((\S+)\/100\%\)/){
5888                 my $percent = $1;
5889                 my $transferred = int($size * $percent / 100);
5890                 my $remaining = $size - $transferred;
5891
5892                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5893             }
5894
5895         };
5896
5897         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5898         my $err = $@;
5899         die "copy failed: $err" if $err;
5900     }
5901 }
5902
5903 sub qemu_img_format {
5904     my ($scfg, $volname) = @_;
5905
5906     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5907         return $1;
5908     } else {
5909         return "raw";
5910     }
5911 }
5912
5913 sub qemu_drive_mirror {
5914     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5915
5916     my $storecfg = PVE::Storage::config();
5917     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5918
5919     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5920
5921     my $format = qemu_img_format($dst_scfg, $dst_volname);
5922
5923     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5924
5925     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5926
5927     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5928     $opts->{format} = $format if $format;
5929
5930     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5931
5932     eval {
5933     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5934         while (1) {
5935             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5936             my $stat = @$stats[0];
5937             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5938             die "error job is not mirroring" if $stat->{type} ne "mirror";
5939
5940             my $busy = $stat->{busy};
5941             my $ready = $stat->{ready};
5942
5943             if (my $total = $stat->{len}) {
5944                 my $transferred = $stat->{offset} || 0;
5945                 my $remaining = $total - $transferred;
5946                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5947
5948                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5949             }
5950
5951
5952             if ($stat->{ready} eq 'true') {
5953
5954                 last if $vmiddst != $vmid;
5955
5956                 # try to switch the disk if source and destination are on the same guest
5957                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5958                 last if !$@;
5959                 die $@ if $@ !~ m/cannot be completed/;
5960             }
5961             sleep 1;
5962         }
5963
5964
5965     };
5966     my $err = $@;
5967
5968     my $cancel_job = sub {
5969         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5970         while (1) {
5971             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5972             my $stat = @$stats[0];
5973             last if !$stat;
5974             sleep 1;
5975         }
5976     };
5977
5978     if ($err) {
5979         eval { &$cancel_job(); };
5980         die "mirroring error: $err";
5981     }
5982
5983     if ($vmiddst != $vmid) {
5984         # if we clone a disk for a new target vm, we don't switch the disk
5985         &$cancel_job(); # so we call block-job-cancel
5986     }
5987 }
5988
5989 sub clone_disk {
5990     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5991         $newvmid, $storage, $format, $full, $newvollist) = @_;
5992
5993     my $newvolid;
5994
5995     if (!$full) {
5996         print "create linked clone of drive $drivename ($drive->{file})\n";
5997         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5998         push @$newvollist, $newvolid;
5999     } else {
6000         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6001         $storeid = $storage if $storage;
6002
6003         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6004         if (!$format) {
6005             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6006             $format = qemu_img_format($scfg, $volname);
6007         }
6008
6009         # test if requested format is supported - else use default
6010         my $supported = grep { $_ eq $format } @$validFormats;
6011         $format = $defFormat if !$supported;
6012
6013         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6014
6015         print "create full clone of drive $drivename ($drive->{file})\n";
6016         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6017         push @$newvollist, $newvolid;
6018
6019         PVE::Storage::activate_volumes($storecfg, $newvollist);
6020
6021         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6022         if (!$running || $snapname) {
6023             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6024         } else {
6025             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
6026         }
6027     }
6028
6029     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6030
6031     my $disk = $drive;
6032     $disk->{format} = undef;
6033     $disk->{file} = $newvolid;
6034     $disk->{size} = $size;
6035
6036     return $disk;
6037 }
6038
6039 # this only works if VM is running
6040 sub get_current_qemu_machine {
6041     my ($vmid) = @_;
6042
6043     my $cmd = { execute => 'query-machines', arguments => {} };
6044     my $res = vm_qmp_command($vmid, $cmd);
6045
6046     my ($current, $default);
6047     foreach my $e (@$res) {
6048         $default = $e->{name} if $e->{'is-default'};
6049         $current = $e->{name} if $e->{'is-current'};
6050     }
6051
6052     # fallback to the default machine if current is not supported by qemu
6053     return $current || $default || 'pc';
6054 }
6055
6056 sub qemu_machine_feature_enabled {
6057     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6058
6059     my $current_major;
6060     my $current_minor;
6061
6062     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6063
6064         $current_major = $3;
6065         $current_minor = $4;
6066
6067     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6068
6069         $current_major = $1;
6070         $current_minor = $2;
6071     }
6072
6073     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6074
6075
6076 }
6077
6078 sub qemu_machine_pxe {
6079     my ($vmid, $conf, $machine) = @_;
6080
6081     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6082
6083     foreach my $opt (keys %$conf) {
6084         next if $opt !~ m/^net(\d+)$/;
6085         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6086         next if !$net;
6087         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6088         return $machine.".pxe" if $romfile =~ m/pxe/;
6089         last;
6090     }
6091
6092     return $machine;
6093 }
6094
6095 sub qemu_use_old_bios_files {
6096     my ($machine_type) = @_;
6097
6098     return if !$machine_type;
6099
6100     my $use_old_bios_files = undef;
6101
6102     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6103         $machine_type = $1;
6104         $use_old_bios_files = 1;
6105     } else {
6106         my $kvmver = kvm_user_version();
6107         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6108         # load new efi bios files on migration. So this hack is required to allow
6109         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6110         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6111         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6112     }
6113
6114     return ($use_old_bios_files, $machine_type);
6115 }
6116
6117 sub lspci {
6118
6119     my $devices = {};
6120
6121     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6122             my (undef, $id, $function) = @_;
6123             my $res = { id => $id, function => $function};
6124             push @{$devices->{$id}}, $res;
6125     });
6126
6127     return $devices;
6128 }
6129
6130 sub vm_iothreads_list {
6131     my ($vmid) = @_;
6132
6133     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6134
6135     my $iothreads = {};
6136     foreach my $iothread (@$res) {
6137         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6138     }
6139
6140     return $iothreads;
6141 }
6142
6143 sub scsihw_infos {
6144     my ($conf, $drive) = @_;
6145
6146     my $maxdev = 0;
6147
6148     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6149         $maxdev = 7;
6150     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6151         $maxdev = 1;
6152     } else {
6153         $maxdev = 256;
6154     }
6155
6156     my $controller = int($drive->{index} / $maxdev);
6157     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6158
6159     return ($maxdev, $controller, $controller_prefix);
6160 }
6161
6162 # bash completion helper
6163
6164 sub complete_backup_archives {
6165     my ($cmdname, $pname, $cvalue) = @_;
6166
6167     my $cfg = PVE::Storage::config();
6168
6169     my $storeid;
6170
6171     if ($cvalue =~ m/^([^:]+):/) {
6172         $storeid = $1;
6173     }
6174
6175     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6176
6177     my $res = [];
6178     foreach my $id (keys %$data) {
6179         foreach my $item (@{$data->{$id}}) {
6180             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6181             push @$res, $item->{volid} if defined($item->{volid});
6182         }
6183     }
6184
6185     return $res;
6186 }
6187
6188 my $complete_vmid_full = sub {
6189     my ($running) = @_;
6190
6191     my $idlist = vmstatus();
6192
6193     my $res = [];
6194
6195     foreach my $id (keys %$idlist) {
6196         my $d = $idlist->{$id};
6197         if (defined($running)) {
6198             next if $d->{template};
6199             next if $running && $d->{status} ne 'running';
6200             next if !$running && $d->{status} eq 'running';
6201         }
6202         push @$res, $id;
6203
6204     }
6205     return $res;
6206 };
6207
6208 sub complete_vmid {
6209     return &$complete_vmid_full();
6210 }
6211
6212 sub complete_vmid_stopped {
6213     return &$complete_vmid_full(0);
6214 }
6215
6216 sub complete_vmid_running {
6217     return &$complete_vmid_full(1);
6218 }
6219
6220 sub complete_storage {
6221
6222     my $cfg = PVE::Storage::config();
6223     my $ids = $cfg->{ids};
6224
6225     my $res = [];
6226     foreach my $sid (keys %$ids) {
6227         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6228         next if !$ids->{$sid}->{content}->{images};
6229         push @$res, $sid;
6230     }
6231
6232     return $res;
6233 }
6234
6235 1;