PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35 use URI::Escape;
  36
  37 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  38
  39 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  40
  41 # Note about locking: we use flock on the config file protect
  42 # against concurent actions.
  43 # Aditionaly, we have a 'lock' setting in the config file. This
  44 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  45 # allowed when such lock is set. But you can ignore this kind of
  46 # lock with the --skiplock flag.
  47
  48 cfs_register_file('/qemu-server/',
  49                   \&parse_vm_config,
  50                   \&write_vm_config);
  51
  52 PVE::JSONSchema::register_standard_option('skiplock', {
  53     description => "Ignore locks - only root is allowed to use this option.",
  54     type => 'boolean',
  55     optional => 1,
  56 });
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 #no warnings 'redefine';
  72
  73 sub cgroups_write {
  74    my ($controller, $vmid, $option, $value) = @_;
  75
  76    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  77    PVE::ProcFSTools::write_proc_entry($path, $value);
  78
  79 }
  80
  81 my $nodename = PVE::INotify::nodename();
  82
  83 mkdir "/etc/pve/nodes/$nodename";
  84 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  85 mkdir $confdir;
  86
  87 my $var_run_tmpdir = "/var/run/qemu-server";
  88 mkdir $var_run_tmpdir;
  89
  90 my $lock_dir = "/var/lock/qemu-server";
  91 mkdir $lock_dir;
  92
  93 my $pcisysfs = "/sys/bus/pci";
  94
  95 my $cpu_vendor_list = {
  96     # Intel CPUs
  97     486 => 'GenuineIntel',
  98     pentium => 'GenuineIntel',
  99     pentium2  => 'GenuineIntel',
 100     pentium3  => 'GenuineIntel',
 101     coreduo => 'GenuineIntel',
 102     core2duo => 'GenuineIntel',
 103     Conroe  => 'GenuineIntel',
 104     Penryn  => 'GenuineIntel',
 105     Nehalem  => 'GenuineIntel',
 106     Westmere => 'GenuineIntel',
 107     SandyBridge => 'GenuineIntel',
 108     IvyBridge => 'GenuineIntel',
 109     Haswell => 'GenuineIntel',
 110     'Haswell-noTSX' => 'GenuineIntel',
 111     Broadwell => 'GenuineIntel',
 112     'Broadwell-noTSX' => 'GenuineIntel',
 113
 114     # AMD CPUs
 115     athlon => 'AuthenticAMD',
 116     phenom  => 'AuthenticAMD',
 117     Opteron_G1  => 'AuthenticAMD',
 118     Opteron_G2  => 'AuthenticAMD',
 119     Opteron_G3  => 'AuthenticAMD',
 120     Opteron_G4  => 'AuthenticAMD',
 121     Opteron_G5  => 'AuthenticAMD',
 122
 123     # generic types, use vendor from host node
 124     host => 'default',
 125     kvm32 => 'default',
 126     kvm64 => 'default',
 127     qemu32 => 'default',
 128     qemu64 => 'default',
 129 };
 130
 131 my $cpu_fmt = {
 132     cputype => {
 133         description => "Emulated CPU type.",
 134         type => 'string',
 135         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 136         format_description => 'cputype',
 137         default => 'kvm64',
 138         default_key => 1,
 139     },
 140     hidden => {
 141         description => "Do not identify as a KVM virtual machine.",
 142         type => 'boolean',
 143         optional => 1,
 144         default => 0
 145     },
 146 };
 147
 148 my $watchdog_fmt = {
 149     model => {
 150         default_key => 1,
 151         type => 'string',
 152         enum => [qw(i6300esb ib700)],
 153         description => "Watchdog type to emulate.",
 154         default => 'i6300esb',
 155         optional => 1,
 156     },
 157     action => {
 158         type => 'string',
 159         enum => [qw(reset shutdown poweroff pause debug none)],
 160         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 161         optional => 1,
 162     },
 163 };
 164 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 165
 166 my $confdesc = {
 167     onboot => {
 168         optional => 1,
 169         type => 'boolean',
 170         description => "Specifies whether a VM will be started during system bootup.",
 171         default => 0,
 172     },
 173     autostart => {
 174         optional => 1,
 175         type => 'boolean',
 176         description => "Automatic restart after crash (currently ignored).",
 177         default => 0,
 178     },
 179     hotplug => {
 180         optional => 1,
 181         type => 'string', format => 'pve-hotplug-features',
 182         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 183         default => 'network,disk,usb',
 184     },
 185     reboot => {
 186         optional => 1,
 187         type => 'boolean',
 188         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 189         default => 1,
 190     },
 191     lock => {
 192         optional => 1,
 193         type => 'string',
 194         description => "Lock/unlock the VM.",
 195         enum => [qw(migrate backup snapshot rollback)],
 196     },
 197     cpulimit => {
 198         optional => 1,
 199         type => 'number',
 200         description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 201         minimum => 0,
 202         maximum => 128,
 203         default => 0,
 204     },
 205     cpuunits => {
 206         optional => 1,
 207         type => 'integer',
 208         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 209         minimum => 0,
 210         maximum => 500000,
 211         default => 1000,
 212     },
 213     memory => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 217         minimum => 16,
 218         default => 512,
 219     },
 220     balloon => {
 221         optional => 1,
 222         type => 'integer',
 223         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 224         minimum => 0,
 225     },
 226     shares => {
 227         optional => 1,
 228         type => 'integer',
 229         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 230         minimum => 0,
 231         maximum => 50000,
 232         default => 1000,
 233     },
 234     keyboard => {
 235         optional => 1,
 236         type => 'string',
 237         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 238         enum => PVE::Tools::kvmkeymaplist(),
 239         default => 'en-us',
 240     },
 241     name => {
 242         optional => 1,
 243         type => 'string', format => 'dns-name',
 244         description => "Set a name for the VM. Only used on the configuration web interface.",
 245     },
 246     scsihw => {
 247         optional => 1,
 248         type => 'string',
 249         description => "scsi controller model",
 250         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 251         default => 'lsi',
 252     },
 253     description => {
 254         optional => 1,
 255         type => 'string',
 256         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 257     },
 258     ostype => {
 259         optional => 1,
 260         type => 'string',
 261         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 262         description => <<EODESC,
 263 Used to enable special optimization/features for specific
 264 operating systems:
 265
 266 other  => unspecified OS
 267 wxp    => Microsoft Windows XP
 268 w2k    => Microsoft Windows 2000
 269 w2k3   => Microsoft Windows 2003
 270 w2k8   => Microsoft Windows 2008
 271 wvista => Microsoft Windows Vista
 272 win7   => Microsoft Windows 7
 273 win8   => Microsoft Windows 8/2012
 274 l24    => Linux 2.4 Kernel
 275 l26    => Linux 2.6/3.X Kernel
 276 solaris => solaris/opensolaris/openindiania kernel
 277
 278 other|l24|l26|solaris                       ... no special behaviour
 279 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 280 EODESC
 281     },
 282     boot => {
 283         optional => 1,
 284         type => 'string',
 285         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 286         pattern => '[acdn]{1,4}',
 287         default => 'cdn',
 288     },
 289     bootdisk => {
 290         optional => 1,
 291         type => 'string', format => 'pve-qm-bootdisk',
 292         description => "Enable booting from specified disk.",
 293         pattern => '(ide|sata|scsi|virtio)\d+',
 294     },
 295     smp => {
 296         optional => 1,
 297         type => 'integer',
 298         description => "The number of CPUs. Please use option -sockets instead.",
 299         minimum => 1,
 300         default => 1,
 301     },
 302     sockets => {
 303         optional => 1,
 304         type => 'integer',
 305         description => "The number of CPU sockets.",
 306         minimum => 1,
 307         default => 1,
 308     },
 309     cores => {
 310         optional => 1,
 311         type => 'integer',
 312         description => "The number of cores per socket.",
 313         minimum => 1,
 314         default => 1,
 315     },
 316     numa => {
 317         optional => 1,
 318         type => 'boolean',
 319         description => "Enable/disable NUMA.",
 320         default => 0,
 321     },
 322     vcpus => {
 323         optional => 1,
 324         type => 'integer',
 325         description => "Number of hotplugged vcpus.",
 326         minimum => 1,
 327         default => 0,
 328     },
 329     acpi => {
 330         optional => 1,
 331         type => 'boolean',
 332         description => "Enable/disable ACPI.",
 333         default => 1,
 334     },
 335     agent => {
 336         optional => 1,
 337         type => 'boolean',
 338         description => "Enable/disable Qemu GuestAgent.",
 339         default => 0,
 340     },
 341     kvm => {
 342         optional => 1,
 343         type => 'boolean',
 344         description => "Enable/disable KVM hardware virtualization.",
 345         default => 1,
 346     },
 347     tdf => {
 348         optional => 1,
 349         type => 'boolean',
 350         description => "Enable/disable time drift fix.",
 351         default => 0,
 352     },
 353     localtime => {
 354         optional => 1,
 355         type => 'boolean',
 356         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 357     },
 358     freeze => {
 359         optional => 1,
 360         type => 'boolean',
 361         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 362     },
 363     vga => {
 364         optional => 1,
 365         type => 'string',
 366         description => "Select the VGA type. If you want to use high resolution" .
 367             " modes (>= 1280x1024x16) then you should use the options " .
 368             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 369             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 370             "display sever. For win* OS you can select how many independent " .
 371             "displays you want, Linux guests can add displays them self. " .
 372             "You can also run without any graphic card, using a serial device" .
 373             " as terminal.",
 374         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 375     },
 376     watchdog => {
 377         optional => 1,
 378         type => 'string', format => 'pve-qm-watchdog',
 379         description => "Create a virtual hardware watchdog device. Once enabled" .
 380             " (by a guest action), the watchdog must be periodically polled " .
 381             "by an agent inside the guest or else the watchdog will reset " .
 382             "the guest (or execute the respective action specified)",
 383     },
 384     startdate => {
 385         optional => 1,
 386         type => 'string',
 387         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 388         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 389         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 390         default => 'now',
 391     },
 392     startup =>  get_standard_option('pve-startup-order'),
 393     template => {
 394         optional => 1,
 395         type => 'boolean',
 396         description => "Enable/disable Template.",
 397         default => 0,
 398     },
 399     args => {
 400         optional => 1,
 401         type => 'string',
 402         description => <<EODESCR,
 403 Arbitrary arguments passed to kvm, for example:
 404
 405 args: -no-reboot -no-hpet
 406
 407 NOTE: this option is for experts only.
 408 EODESCR
 409     },
 410     tablet => {
 411         optional => 1,
 412         type => 'boolean',
 413         default => 1,
 414         description => "Enable/disable the USB tablet device. This device is " .
 415             "usually needed to allow absolute mouse positioning with VNC. " .
 416             "Else the mouse runs out of sync with normal VNC clients. " .
 417             "If you're running lots of console-only guests on one host, " .
 418             "you may consider disabling this to save some context switches. " .
 419             "This is turned off by default if you use spice (-vga=qxl).",
 420     },
 421     migrate_speed => {
 422         optional => 1,
 423         type => 'integer',
 424         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 425         minimum => 0,
 426         default => 0,
 427     },
 428     migrate_downtime => {
 429         optional => 1,
 430         type => 'number',
 431         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 432         minimum => 0,
 433         default => 0.1,
 434     },
 435     cdrom => {
 436         optional => 1,
 437         type => 'string', format => 'pve-qm-drive',
 438         typetext => 'volume',
 439         description => "This is an alias for option -ide2",
 440     },
 441     cpu => {
 442         optional => 1,
 443         description => "Emulated CPU type.",
 444         type => 'string',
 445         format => $cpu_fmt,
 446     },
 447     parent => get_standard_option('pve-snapshot-name', {
 448         optional => 1,
 449         description => "Parent snapshot name. This is used internally, and should not be modified.",
 450     }),
 451     snaptime => {
 452         optional => 1,
 453         description => "Timestamp for snapshots.",
 454         type => 'integer',
 455         minimum => 0,
 456     },
 457     vmstate => {
 458         optional => 1,
 459         type => 'string', format => 'pve-volume-id',
 460         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 461     },
 462     machine => {
 463         description => "Specific the Qemu machine type.",
 464         type => 'string',
 465         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 466         maxLength => 40,
 467         optional => 1,
 468     },
 469     smbios1 => {
 470         description => "Specify SMBIOS type 1 fields.",
 471         type => 'string', format => 'pve-qm-smbios1',
 472         maxLength => 256,
 473         optional => 1,
 474     },
 475     protection => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 479         default => 0,
 480     },
 481     bios => {
 482         optional => 1,
 483         type => 'string',
 484         enum => [ qw(seabios ovmf) ],
 485         description => "Select BIOS implementation.",
 486         default => 'seabios',
 487     },
 488 };
 489
 490 # what about other qemu settings ?
 491 #cpu => 'string',
 492 #machine => 'string',
 493 #fda => 'file',
 494 #fdb => 'file',
 495 #mtdblock => 'file',
 496 #sd => 'file',
 497 #pflash => 'file',
 498 #snapshot => 'bool',
 499 #bootp => 'file',
 500 ##tftp => 'dir',
 501 ##smb => 'dir',
 502 #kernel => 'file',
 503 #append => 'string',
 504 #initrd => 'file',
 505 ##soundhw => 'string',
 506
 507 while (my ($k, $v) = each %$confdesc) {
 508     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 509 }
 510
 511 my $MAX_IDE_DISKS = 4;
 512 my $MAX_SCSI_DISKS = 14;
 513 my $MAX_VIRTIO_DISKS = 16;
 514 my $MAX_SATA_DISKS = 6;
 515 my $MAX_USB_DEVICES = 5;
 516 my $MAX_NETS = 32;
 517 my $MAX_UNUSED_DISKS = 8;
 518 my $MAX_HOSTPCI_DEVICES = 4;
 519 my $MAX_SERIAL_PORTS = 4;
 520 my $MAX_PARALLEL_PORTS = 3;
 521 my $MAX_NUMA = 8;
 522 my $MAX_MEM = 4194304;
 523 my $STATICMEM = 1024;
 524
 525 my $numa_fmt = {
 526     cpus => {
 527         type => "string",
 528         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 529         description => "CPUs accessing this numa node.",
 530         format_description => "id[-id];...",
 531     },
 532     memory => {
 533         type => "number",
 534         description => "Amount of memory this numa node provides.",
 535         optional => 1,
 536     },
 537     hostnodes => {
 538         type => "string",
 539         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 540         description => "host numa nodes to use",
 541         format_description => "id[-id];...",
 542         optional => 1,
 543     },
 544     policy => {
 545         type => 'string',
 546         enum => [qw(preferred bind interleave)],
 547         description => "numa allocation policy.",
 548         optional => 1,
 549     },
 550 };
 551 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 552 my $numadesc = {
 553     optional => 1,
 554     type => 'string', format => $numa_fmt,
 555     description => "numa topology",
 556 };
 557 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 558
 559 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 560     $confdesc->{"numa$i"} = $numadesc;
 561 }
 562
 563 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 564                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 565                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 566 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 567
 568 my $net_fmt = {
 569     macaddr => {
 570         type => 'string',
 571         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 572         description => "MAC address",
 573         format_description => "XX:XX:XX:XX:XX:XX",
 574         optional => 1,
 575     },
 576     model => {
 577         type => 'string',
 578         description => 'Network Card Model.',
 579         format_description => 'model',
 580         enum => $nic_model_list,
 581         default_key => 1,
 582     },
 583     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 584     bridge => {
 585         type => 'string',
 586         description => 'Bridge to attach the network device to.',
 587         format_description => 'bridge',
 588         optional => 1,
 589     },
 590     queues => {
 591         type => 'integer',
 592         minimum => 0, maximum => 16,
 593         description => 'Number of packet queues to be used on the device.',
 594         optional => 1,
 595     },
 596     rate => {
 597         type => 'number',
 598         minimum => 0,
 599         description => 'Rate limit in mbps as floating point number.',
 600         optional => 1,
 601     },
 602     tag => {
 603         type => 'integer',
 604         minimum => 2, maximum => 4094,
 605         description => 'VLAN tag to apply to packets on this interface.',
 606         optional => 1,
 607     },
 608     trunks => {
 609         type => 'string',
 610         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 611         description => 'VLAN trunks to pass through this interface.',
 612         format_description => 'vlanid[;vlanid...]',
 613         optional => 1,
 614     },
 615     firewall => {
 616         type => 'boolean',
 617         description => 'Whether this interface should be protected by the firewall.',
 618         optional => 1,
 619     },
 620     link_down => {
 621         type => 'boolean',
 622         description => 'Whether this interface should be DISconnected (like pulling the plug).',
 623         optional => 1,
 624     },
 625 };
 626 my $netdesc = {
 627     optional => 1,
 628     type => 'string', format => $net_fmt,
 629     description => <<EODESCR,
 630 Specify network devices.
 631
 632 MODEL is one of: $nic_model_list_txt
 633
 634 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 635 automatically generated if not specified.
 636
 637 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 638
 639 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 640
 641 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 642
 643 10.0.2.2   Gateway
 644 10.0.2.3   DNS Server
 645 10.0.2.4   SMB Server
 646
 647 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 648
 649 EODESCR
 650 };
 651 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 652
 653 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 654     $confdesc->{"net$i"} = $netdesc;
 655 }
 656
 657 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 658 sub verify_volume_id_or_qm_path {
 659     my ($volid, $noerr) = @_;
 660
 661     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 662         return $volid;
 663     }
 664
 665     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 666     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 667     if ($@) {
 668         return undef if $noerr;
 669         die $@;
 670     }
 671     return $volid;
 672 }
 673
 674 my $drivename_hash;
 675
 676 my %drivedesc_base = (
 677     volume => { alias => 'file' },
 678     file => {
 679         type => 'string',
 680         format => 'pve-volume-id-or-qm-path',
 681         default_key => 1,
 682         format_description => 'volume',
 683         description => "The drive's backing volume.",
 684     },
 685     media => {
 686         type => 'string',
 687         enum => [qw(cdrom disk)],
 688         description => "The drive's media type.",
 689         default => 'disk',
 690         optional => 1
 691     },
 692     cyls => {
 693         type => 'integer',
 694         description => "Force the drive's physical geometry to have a specific cylinder count.",
 695         optional => 1
 696     },
 697     heads => {
 698         type => 'integer',
 699         description => "Force the drive's physical geometry to have a specific head count.",
 700         optional => 1
 701     },
 702     secs => {
 703         type => 'integer',
 704         description => "Force the drive's physical geometry to have a specific sector count.",
 705         optional => 1
 706     },
 707     trans => {
 708         type => 'string',
 709         enum => [qw(none lba auto)],
 710         description => "Force disk geometry bios translation mode.",
 711         optional => 1,
 712     },
 713     snapshot => {
 714         type => 'boolean',
 715         description => "Whether the drive should be included when making snapshots.",
 716         optional => 1,
 717     },
 718     cache => {
 719         type => 'string',
 720         enum => [qw(none writethrough writeback unsafe directsync)],
 721         description => "The drive's cache mode",
 722         optional => 1,
 723     },
 724     format => {
 725         type => 'string',
 726         format_description => 'image format',
 727         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 728         description => "The drive's backing file's data format.",
 729         optional => 1,
 730     },
 731     size => {
 732         type => 'string',
 733         format => 'disk-size',
 734         format_description => 'DiskSize',
 735         description => "Disk size. This is purely informational and has no effect.",
 736         optional => 1,
 737     },
 738     backup => {
 739         type => 'boolean',
 740         description => "Whether the drive should be included when making backups.",
 741         optional => 1,
 742     },
 743     werror => {
 744         type => 'string',
 745         enum => [qw(enospc ignore report stop)],
 746         description => 'Write error action.',
 747         optional => 1,
 748     },
 749     aio => {
 750         type => 'string',
 751         enum => [qw(native threads)],
 752         description => 'AIO type to use.',
 753         optional => 1,
 754     },
 755     discard => {
 756         type => 'string',
 757         enum => [qw(ignore on)],
 758         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 759         optional => 1,
 760     },
 761     detect_zeroes => {
 762         type => 'boolean',
 763         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 764         optional => 1,
 765     },
 766     serial => {
 767         type => 'string',
 768         format => 'urlencoded',
 769         format_description => 'serial',
 770         maxLength => 20*3, # *3 since it's %xx url enoded
 771         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 772         optional => 1,
 773     }
 774 );
 775
 776 my %rerror_fmt = (
 777     rerror => {
 778         type => 'string',
 779         enum => [qw(ignore report stop)],
 780         description => 'Read error action.',
 781         optional => 1,
 782     },
 783 );
 784
 785 my %iothread_fmt = ( iothread => {
 786         type => 'boolean',
 787         description => "Whether to use iothreads for this drive",
 788         optional => 1,
 789 });
 790
 791 my %model_fmt = (
 792     model => {
 793         type => 'string',
 794         format => 'urlencoded',
 795         format_description => 'model',
 796         maxLength => 40*3, # *3 since it's %xx url enoded
 797         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 798         optional => 1,
 799     },
 800 );
 801
 802 my %queues_fmt = (
 803     queues => {
 804         type => 'integer',
 805         description => "Number of queues.",
 806         minimum => 2,
 807         optional => 1
 808     }
 809 );
 810
 811 my $add_throttle_desc = sub {
 812     my ($key, $type, $what, $unit, $longunit) = @_;
 813     $drivedesc_base{$key} = {
 814         type => $type,
 815         format_description => $unit,
 816         description => "Maximum $what speed in $longunit per second.",
 817         optional => 1,
 818     };
 819 };
 820 # throughput: (leaky bucket)
 821 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 822 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 823 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 824 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 825 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 826 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 827 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 828 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 829 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 830
 831 # pools: (pool of IO before throttling starts taking effect)
 832 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 833 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 834 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 835 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 836 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 837 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 838
 839 my $ide_fmt = {
 840     %drivedesc_base,
 841     %rerror_fmt,
 842     %model_fmt,
 843 };
 844
 845 my $idedesc = {
 846     optional => 1,
 847     type => 'string', format => $ide_fmt,
 848     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 849 };
 850 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 851
 852 my $scsi_fmt = {
 853     %drivedesc_base,
 854     %iothread_fmt,
 855     %queues_fmt,
 856 };
 857 my $scsidesc = {
 858     optional => 1,
 859     type => 'string', format => $scsi_fmt,
 860     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 861 };
 862 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 863
 864 my $sata_fmt = {
 865     %drivedesc_base,
 866     %rerror_fmt,
 867 };
 868 my $satadesc = {
 869     optional => 1,
 870     type => 'string', format => $sata_fmt,
 871     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 872 };
 873 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 874
 875 my $virtio_fmt = {
 876     %drivedesc_base,
 877     %iothread_fmt,
 878     %rerror_fmt,
 879 };
 880 my $virtiodesc = {
 881     optional => 1,
 882     type => 'string', format => $virtio_fmt,
 883     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 884 };
 885 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 886
 887 my $alldrive_fmt = {
 888     %drivedesc_base,
 889     %rerror_fmt,
 890     %iothread_fmt,
 891     %model_fmt,
 892     %queues_fmt,
 893 };
 894
 895 my $usb_fmt = {
 896     host => {
 897         default_key => 1,
 898         type => 'string', format => 'pve-qm-usb-device',
 899         format_description => 'HOSTUSBDEVICE|spice',
 900         description => 'The Host USB device or port or the value spice',
 901     },
 902     usb3 => {
 903         optional => 1,
 904         type => 'boolean',
 905 #       format_description => 'yes|no',
 906         description => 'Specifies whether if given host option is a USB3 device or port',
 907     },
 908 };
 909
 910 my $usbdesc = {
 911     optional => 1,
 912     type => 'string', format => $usb_fmt,
 913     description => <<EODESCR,
 914 Configure an USB device (n is 0 to 4). This can be used to
 915 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 916
 917 'bus-port(.port)*' (decimal numbers) or
 918 'vendor_id:product_id' (hexadeciaml numbers) or
 919 'spice'
 920
 921 You can use the 'lsusb -t' command to list existing usb devices.
 922
 923 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 924
 925 The value 'spice' can be used to add a usb redirection devices for spice.
 926
 927 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 928
 929 EODESCR
 930 };
 931 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 932
 933 # NOTE: the match-groups of this regex are used in parse_hostpci
 934 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 935 my $hostpci_fmt = {
 936     host => {
 937         default_key => 1,
 938         type => 'string',
 939         pattern => qr/$PCIRE(;$PCIRE)*/,
 940         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 941         description => "The PCI ID of a host's PCI device or a list of PCI virtual functions of the host.",
 942     },
 943     rombar => {
 944         type => 'boolean',
 945         optional => 1,
 946         default => 1,
 947     },
 948     pcie => {
 949         type => 'boolean',
 950         optional => 1,
 951         default => 0,
 952     },
 953     'x-vga' => {
 954         type => 'boolean',
 955         optional => 1,
 956         default => 0,
 957     },
 958 };
 959 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
 960
 961 my $hostpcidesc = {
 962         optional => 1,
 963         type => 'string', format => 'pve-qm-hostpci',
 964         description => <<EODESCR,
 965 Map host pci devices. HOSTPCIDEVICE syntax is:
 966
 967 'bus:dev.func' (hexadecimal numbers)
 968
 969 You can us the 'lspci' command to list existing pci devices.
 970
 971 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 972
 973 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 974
 975 Experimental: user reported problems with this option.
 976 EODESCR
 977 };
 978 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 979
 980 my $serialdesc = {
 981         optional => 1,
 982         type => 'string',
 983         pattern => '(/dev/.+|socket)',
 984         description =>  <<EODESCR,
 985 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 986
 987 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 988
 989 Experimental: user reported problems with this option.
 990 EODESCR
 991 };
 992
 993 my $paralleldesc= {
 994         optional => 1,
 995         type => 'string',
 996         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 997         description =>  <<EODESCR,
 998 Map host parallel devices (n is 0 to 2).
 999
1000 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1001
1002 Experimental: user reported problems with this option.
1003 EODESCR
1004 };
1005
1006 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1007     $confdesc->{"parallel$i"} = $paralleldesc;
1008 }
1009
1010 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1011     $confdesc->{"serial$i"} = $serialdesc;
1012 }
1013
1014 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1015     $confdesc->{"hostpci$i"} = $hostpcidesc;
1016 }
1017
1018 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1019     $drivename_hash->{"ide$i"} = 1;
1020     $confdesc->{"ide$i"} = $idedesc;
1021 }
1022
1023 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1024     $drivename_hash->{"sata$i"} = 1;
1025     $confdesc->{"sata$i"} = $satadesc;
1026 }
1027
1028 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1029     $drivename_hash->{"scsi$i"} = 1;
1030     $confdesc->{"scsi$i"} = $scsidesc ;
1031 }
1032
1033 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1034     $drivename_hash->{"virtio$i"} = 1;
1035     $confdesc->{"virtio$i"} = $virtiodesc;
1036 }
1037
1038 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1039     $confdesc->{"usb$i"} = $usbdesc;
1040 }
1041
1042 my $unuseddesc = {
1043     optional => 1,
1044     type => 'string', format => 'pve-volume-id',
1045     description => "Reference to unused volumes.",
1046 };
1047
1048 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1049     $confdesc->{"unused$i"} = $unuseddesc;
1050 }
1051
1052 my $kvm_api_version = 0;
1053
1054 sub kvm_version {
1055
1056     return $kvm_api_version if $kvm_api_version;
1057
1058     my $fh = IO::File->new("</dev/kvm") ||
1059         return 0;
1060
1061     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1062         $kvm_api_version = $v;
1063     }
1064
1065     $fh->close();
1066
1067     return  $kvm_api_version;
1068 }
1069
1070 my $kvm_user_version;
1071
1072 sub kvm_user_version {
1073
1074     return $kvm_user_version if $kvm_user_version;
1075
1076     $kvm_user_version = 'unknown';
1077
1078     my $code = sub {
1079         my $line = shift;
1080         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1081             $kvm_user_version = $2;
1082         }
1083     };
1084
1085     eval { run_command("kvm -version", outfunc => $code); };
1086     warn $@ if $@;
1087
1088     return $kvm_user_version;
1089
1090 }
1091
1092 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1093
1094 sub valid_drive_names {
1095     # order is important - used to autoselect boot disk
1096     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1097             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1098             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1099             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1100 }
1101
1102 sub is_valid_drivename {
1103     my $dev = shift;
1104
1105     return defined($drivename_hash->{$dev});
1106 }
1107
1108 sub option_exists {
1109     my $key = shift;
1110     return defined($confdesc->{$key});
1111 }
1112
1113 sub nic_models {
1114     return $nic_model_list;
1115 }
1116
1117 sub os_list_description {
1118
1119     return {
1120         other => 'Other',
1121         wxp => 'Windows XP',
1122         w2k => 'Windows 2000',
1123         w2k3 =>, 'Windows 2003',
1124         w2k8 => 'Windows 2008',
1125         wvista => 'Windows Vista',
1126         win7 => 'Windows 7',
1127         win8 => 'Windows 8/2012',
1128         l24 => 'Linux 2.4',
1129         l26 => 'Linux 2.6',
1130     };
1131 }
1132
1133 my $cdrom_path;
1134
1135 sub get_cdrom_path {
1136
1137     return  $cdrom_path if $cdrom_path;
1138
1139     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1140     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1141     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1142 }
1143
1144 sub get_iso_path {
1145     my ($storecfg, $vmid, $cdrom) = @_;
1146
1147     if ($cdrom eq 'cdrom') {
1148         return get_cdrom_path();
1149     } elsif ($cdrom eq 'none') {
1150         return '';
1151     } elsif ($cdrom =~ m|^/|) {
1152         return $cdrom;
1153     } else {
1154         return PVE::Storage::path($storecfg, $cdrom);
1155     }
1156 }
1157
1158 # try to convert old style file names to volume IDs
1159 sub filename_to_volume_id {
1160     my ($vmid, $file, $media) = @_;
1161
1162     if (!($file eq 'none' || $file eq 'cdrom' ||
1163           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1164
1165         return undef if $file =~ m|/|;
1166
1167         if ($media && $media eq 'cdrom') {
1168             $file = "local:iso/$file";
1169         } else {
1170             $file = "local:$vmid/$file";
1171         }
1172     }
1173
1174     return $file;
1175 }
1176
1177 sub verify_media_type {
1178     my ($opt, $vtype, $media) = @_;
1179
1180     return if !$media;
1181
1182     my $etype;
1183     if ($media eq 'disk') {
1184         $etype = 'images';
1185     } elsif ($media eq 'cdrom') {
1186         $etype = 'iso';
1187     } else {
1188         die "internal error";
1189     }
1190
1191     return if ($vtype eq $etype);
1192
1193     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1194 }
1195
1196 sub cleanup_drive_path {
1197     my ($opt, $storecfg, $drive) = @_;
1198
1199     # try to convert filesystem paths to volume IDs
1200
1201     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1202         ($drive->{file} !~ m|^/dev/.+|) &&
1203         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1204         ($drive->{file} !~ m/^\d+$/)) {
1205         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1206         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1207         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1208         verify_media_type($opt, $vtype, $drive->{media});
1209         $drive->{file} = $volid;
1210     }
1211
1212     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1213 }
1214
1215 sub parse_hotplug_features {
1216     my ($data) = @_;
1217
1218     my $res = {};
1219
1220     return $res if $data eq '0';
1221
1222     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1223
1224     foreach my $feature (PVE::Tools::split_list($data)) {
1225         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1226             $res->{$1} = 1;
1227         } else {
1228             warn "ignoring unknown hotplug feature '$feature'\n";
1229         }
1230     }
1231     return $res;
1232 }
1233
1234 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1235 sub pve_verify_hotplug_features {
1236     my ($value, $noerr) = @_;
1237
1238     return $value if parse_hotplug_features($value);
1239
1240     return undef if $noerr;
1241
1242     die "unable to parse hotplug option\n";
1243 }
1244
1245 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1246 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1247 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1248 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1249 #        [,iothread=on][,serial=serial][,model=model]
1250
1251 sub parse_drive {
1252     my ($key, $data) = @_;
1253
1254     my ($interface, $index);
1255
1256     if ($key =~ m/^([^\d]+)(\d+)$/) {
1257         $interface = $1;
1258         $index = $2;
1259     } else {
1260         return undef;
1261     }
1262
1263     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1264                                      : $confdesc->{$key}->{format};
1265     if (!$desc) {
1266         warn "invalid drive key: $key\n";
1267         return undef;
1268     }
1269     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1270     return undef if !$res;
1271     $res->{interface} = $interface;
1272     $res->{index} = $index;
1273
1274     my $error = 0;
1275     foreach my $opt (qw(bps bps_rd bps_wr)) {
1276         if (my $bps = defined(delete $res->{$opt})) {
1277             if (defined($res->{"m$opt"})) {
1278                 warn "both $opt and m$opt specified\n";
1279                 ++$error;
1280                 next;
1281             }
1282             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1283         }
1284     }
1285     return undef if $error;
1286
1287     return undef if $res->{mbps_rd} && $res->{mbps};
1288     return undef if $res->{mbps_wr} && $res->{mbps};
1289     return undef if $res->{iops_rd} && $res->{iops};
1290     return undef if $res->{iops_wr} && $res->{iops};
1291
1292     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1293         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1294         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1295         return undef if $res->{interface} eq 'virtio';
1296     }
1297
1298     if (my $size = $res->{size}) {
1299         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1300     }
1301
1302     return $res;
1303 }
1304
1305 sub print_drive {
1306     my ($vmid, $drive) = @_;
1307     my $data = { %$drive };
1308     delete $data->{$_} for qw(index interface);
1309     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1310 }
1311
1312 sub scsi_inquiry {
1313     my($fh, $noerr) = @_;
1314
1315     my $SG_IO = 0x2285;
1316     my $SG_GET_VERSION_NUM = 0x2282;
1317
1318     my $versionbuf = "\x00" x 8;
1319     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1320     if (!$ret) {
1321         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1322         return undef;
1323     }
1324     my $version = unpack("I", $versionbuf);
1325     if ($version < 30000) {
1326         die "scsi generic interface too old\n"  if !$noerr;
1327         return undef;
1328     }
1329
1330     my $buf = "\x00" x 36;
1331     my $sensebuf = "\x00" x 8;
1332     my $cmd = pack("C x3 C x1", 0x12, 36);
1333
1334     # see /usr/include/scsi/sg.h
1335     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1336
1337     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1338                       length($sensebuf), 0, length($buf), $buf,
1339                       $cmd, $sensebuf, 6000);
1340
1341     $ret = ioctl($fh, $SG_IO, $packet);
1342     if (!$ret) {
1343         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1344         return undef;
1345     }
1346
1347     my @res = unpack($sg_io_hdr_t, $packet);
1348     if ($res[17] || $res[18]) {
1349         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1350         return undef;
1351     }
1352
1353     my $res = {};
1354     (my $byte0, my $byte1, $res->{vendor},
1355      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1356
1357     $res->{removable} = $byte1 & 128 ? 1 : 0;
1358     $res->{type} = $byte0 & 31;
1359
1360     return $res;
1361 }
1362
1363 sub path_is_scsi {
1364     my ($path) = @_;
1365
1366     my $fh = IO::File->new("+<$path") || return undef;
1367     my $res = scsi_inquiry($fh, 1);
1368     close($fh);
1369
1370     return $res;
1371 }
1372
1373 sub machine_type_is_q35 {
1374     my ($conf) = @_;
1375
1376     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1377 }
1378
1379 sub print_tabletdevice_full {
1380     my ($conf) = @_;
1381
1382     my $q35 = machine_type_is_q35($conf);
1383
1384     # we use uhci for old VMs because tablet driver was buggy in older qemu
1385     my $usbbus = $q35 ? "ehci" : "uhci";
1386
1387     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1388 }
1389
1390 sub print_drivedevice_full {
1391     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1392
1393     my $device = '';
1394     my $maxdev = 0;
1395
1396     if ($drive->{interface} eq 'virtio') {
1397         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1398         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1399         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1400     } elsif ($drive->{interface} eq 'scsi') {
1401
1402         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1403         my $unit = $drive->{index} % $maxdev;
1404         my $devicetype = 'hd';
1405         my $path = '';
1406         if (drive_is_cdrom($drive)) {
1407             $devicetype = 'cd';
1408         } else {
1409             if ($drive->{file} =~ m|^/|) {
1410                 $path = $drive->{file};
1411                 if (my $info = path_is_scsi($path)) {
1412                     if ($info->{type} == 0) {
1413                         $devicetype = 'block';
1414                     } elsif ($info->{type} == 1) { # tape
1415                         $devicetype = 'generic';
1416                     }
1417                 }
1418             } else {
1419                  $path = PVE::Storage::path($storecfg, $drive->{file});
1420             }
1421
1422             if($path =~ m/^iscsi\:\/\//){
1423                 $devicetype = 'generic';
1424             }
1425         }
1426
1427         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1428             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1429         } else {
1430             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1431         }
1432
1433     } elsif ($drive->{interface} eq 'ide'){
1434         $maxdev = 2;
1435         my $controller = int($drive->{index} / $maxdev);
1436         my $unit = $drive->{index} % $maxdev;
1437         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1438
1439         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1440         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1441             $model = URI::Escape::uri_unescape($model);
1442             $device .= ",model=$model";
1443         }
1444     } elsif ($drive->{interface} eq 'sata'){
1445         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1446         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1447         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1448     } elsif ($drive->{interface} eq 'usb') {
1449         die "implement me";
1450         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1451     } else {
1452         die "unsupported interface type";
1453     }
1454
1455     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1456
1457     return $device;
1458 }
1459
1460 sub get_initiator_name {
1461     my $initiator;
1462
1463     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1464     while (defined(my $line = <$fh>)) {
1465         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1466         $initiator = $1;
1467         last;
1468     }
1469     $fh->close();
1470
1471     return $initiator;
1472 }
1473
1474 sub print_drive_full {
1475     my ($storecfg, $vmid, $drive) = @_;
1476
1477     my $path;
1478     my $volid = $drive->{file};
1479     my $format;
1480
1481     if (drive_is_cdrom($drive)) {
1482         $path = get_iso_path($storecfg, $vmid, $volid);
1483     } else {
1484         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1485         if ($storeid) {
1486             $path = PVE::Storage::path($storecfg, $volid);
1487             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1488             $format = qemu_img_format($scfg, $volname);
1489         } else {
1490             $path = $volid;
1491             $format = "raw";
1492         }
1493    }
1494
1495     my $opts = '';
1496     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1497     foreach my $o (@qemu_drive_options) {
1498         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1499     }
1500     if (my $serial = $drive->{serial}) {
1501         $serial = URI::Escape::uri_unescape($serial);
1502         $opts .= ",serial=$serial";
1503     }
1504
1505     $opts .= ",format=$format" if $format && !$drive->{format};
1506
1507     foreach my $o (qw(bps bps_rd bps_wr)) {
1508         my $v = $drive->{"m$o"};
1509         $opts .= ",$o=" . int($v*1024*1024) if $v;
1510     }
1511
1512     my $cache_direct = 0;
1513
1514     if (my $cache = $drive->{cache}) {
1515         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1516     } elsif (!drive_is_cdrom($drive)) {
1517         $opts .= ",cache=none";
1518         $cache_direct = 1;
1519     }
1520
1521     # aio native works only with O_DIRECT
1522     if (!$drive->{aio}) {
1523         if($cache_direct) {
1524             $opts .= ",aio=native";
1525         } else {
1526             $opts .= ",aio=threads";
1527         }
1528     }
1529
1530     if (!drive_is_cdrom($drive)) {
1531         my $detectzeroes;
1532         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1533             $detectzeroes = 'off';
1534         } elsif ($drive->{discard}) {
1535             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1536         } else {
1537             # This used to be our default with discard not being specified:
1538             $detectzeroes = 'on';
1539         }
1540         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1541     }
1542
1543     my $pathinfo = $path ? "file=$path," : '';
1544
1545     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1546 }
1547
1548 sub print_netdevice_full {
1549     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1550
1551     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1552
1553     my $device = $net->{model};
1554     if ($net->{model} eq 'virtio') {
1555          $device = 'virtio-net-pci';
1556      };
1557
1558     my $pciaddr = print_pci_addr("$netid", $bridges);
1559     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1560     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1561         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1562         my $vectors = $net->{queues} * 2 + 2;
1563         $tmpstr .= ",vectors=$vectors,mq=on";
1564     }
1565     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1566
1567     if ($use_old_bios_files) {
1568         my $romfile;
1569         if ($device eq 'virtio-net-pci') {
1570             $romfile = 'pxe-virtio.rom';
1571         } elsif ($device eq 'e1000') {
1572             $romfile = 'pxe-e1000.rom';
1573         } elsif ($device eq 'ne2k') {
1574             $romfile = 'pxe-ne2k_pci.rom';
1575         } elsif ($device eq 'pcnet') {
1576             $romfile = 'pxe-pcnet.rom';
1577         } elsif ($device eq 'rtl8139') {
1578             $romfile = 'pxe-rtl8139.rom';
1579         }
1580         $tmpstr .= ",romfile=$romfile" if $romfile;
1581     }
1582
1583     return $tmpstr;
1584 }
1585
1586 sub print_netdev_full {
1587     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1588
1589     my $i = '';
1590     if ($netid =~ m/^net(\d+)$/) {
1591         $i = int($1);
1592     }
1593
1594     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1595
1596     my $ifname = "tap${vmid}i$i";
1597
1598     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1599     die "interface name '$ifname' is too long (max 15 character)\n"
1600         if length($ifname) >= 16;
1601
1602     my $vhostparam = '';
1603     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1604
1605     my $vmname = $conf->{name} || "vm$vmid";
1606
1607     my $netdev = "";
1608     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1609
1610     if ($net->{bridge}) {
1611         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1612     } else {
1613         $netdev = "type=user,id=$netid,hostname=$vmname";
1614     }
1615
1616     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1617
1618     return $netdev;
1619 }
1620
1621 sub drive_is_cdrom {
1622     my ($drive) = @_;
1623
1624     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1625
1626 }
1627
1628 sub parse_number_sets {
1629     my ($set) = @_;
1630     my $res = [];
1631     foreach my $part (split(/;/, $set)) {
1632         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1633             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1634             push @$res, [ $1, $2 ];
1635         } else {
1636             die "invalid range: $part\n";
1637         }
1638     }
1639     return $res;
1640 }
1641
1642 sub parse_numa {
1643     my ($data) = @_;
1644
1645     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1646     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1647     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1648     return $res;
1649 }
1650
1651 sub parse_hostpci {
1652     my ($value) = @_;
1653
1654     return undef if !$value;
1655
1656     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1657
1658     my @idlist = split(/;/, $res->{host});
1659     delete $res->{host};
1660     foreach my $id (@idlist) {
1661         if ($id =~ /^$PCIRE$/) {
1662             push @{$res->{pciid}}, { id => $1, function => ($2//'0') };
1663         } else {
1664             # should have been caught by parse_property_string already
1665             die "failed to parse PCI id: $id\n";
1666         }
1667     }
1668     return $res;
1669 }
1670
1671 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1672 sub parse_net {
1673     my ($data) = @_;
1674
1675     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1676     if ($@) {
1677         warn $@;
1678         return undef;
1679     }
1680     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1681     return $res;
1682 }
1683
1684 sub print_net {
1685     my $net = shift;
1686
1687     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1688 }
1689
1690 sub add_random_macs {
1691     my ($settings) = @_;
1692
1693     foreach my $opt (keys %$settings) {
1694         next if $opt !~ m/^net(\d+)$/;
1695         my $net = parse_net($settings->{$opt});
1696         next if !$net;
1697         $settings->{$opt} = print_net($net);
1698     }
1699 }
1700
1701 sub vm_is_volid_owner {
1702     my ($storecfg, $vmid, $volid) = @_;
1703
1704     if ($volid !~  m|^/|) {
1705         my ($path, $owner);
1706         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1707         if ($owner && ($owner == $vmid)) {
1708             return 1;
1709         }
1710     }
1711
1712     return undef;
1713 }
1714
1715 sub split_flagged_list {
1716     my $text = shift || '';
1717     $text =~ s/[,;]/ /g;
1718     $text =~ s/^\s+//;
1719     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1720 }
1721
1722 sub join_flagged_list {
1723     my ($how, $lst) = @_;
1724     join $how, map { $lst->{$_} . $_ } keys %$lst;
1725 }
1726
1727 sub vmconfig_delete_pending_option {
1728     my ($conf, $key, $force) = @_;
1729
1730     delete $conf->{pending}->{$key};
1731     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1732     $pending_delete_hash->{$key} = $force ? '!' : '';
1733     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1734 }
1735
1736 sub vmconfig_undelete_pending_option {
1737     my ($conf, $key) = @_;
1738
1739     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1740     delete $pending_delete_hash->{$key};
1741
1742     if (%$pending_delete_hash) {
1743         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1744     } else {
1745         delete $conf->{pending}->{delete};
1746     }
1747 }
1748
1749 sub vmconfig_register_unused_drive {
1750     my ($storecfg, $vmid, $conf, $drive) = @_;
1751
1752     if (!drive_is_cdrom($drive)) {
1753         my $volid = $drive->{file};
1754         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1755             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1756         }
1757     }
1758 }
1759
1760 sub vmconfig_cleanup_pending {
1761     my ($conf) = @_;
1762
1763     # remove pending changes when nothing changed
1764     my $changes;
1765     foreach my $opt (keys %{$conf->{pending}}) {
1766         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1767             $changes = 1;
1768             delete $conf->{pending}->{$opt};
1769         }
1770     }
1771
1772     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1773     my $pending_delete_hash = {};
1774     while (my ($opt, $force) = each %$current_delete_hash) {
1775         if (defined($conf->{$opt})) {
1776             $pending_delete_hash->{$opt} = $force;
1777         } else {
1778             $changes = 1;
1779         }
1780     }
1781
1782     if (%$pending_delete_hash) {
1783         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1784     } else {
1785         delete $conf->{pending}->{delete};
1786     }
1787
1788     return $changes;
1789 }
1790
1791 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1792 my $smbios1_fmt = {
1793     uuid => {
1794         type => 'string',
1795         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1796         format_description => 'UUID',
1797         optional => 1,
1798     },
1799     version => {
1800         type => 'string',
1801         pattern => '\S+',
1802         format_description => 'string',
1803         optional => 1,
1804     },
1805     serial => {
1806         type => 'string',
1807         pattern => '\S+',
1808         format_description => 'string',
1809         optional => 1,
1810     },
1811     manufacturer => {
1812         type => 'string',
1813         pattern => '\S+',
1814         format_description => 'string',
1815         optional => 1,
1816     },
1817     product => {
1818         type => 'string',
1819         pattern => '\S+',
1820         format_description => 'string',
1821         optional => 1,
1822     },
1823     sku => {
1824         type => 'string',
1825         pattern => '\S+',
1826         format_description => 'string',
1827         optional => 1,
1828     },
1829     family => {
1830         type => 'string',
1831         pattern => '\S+',
1832         format_description => 'string',
1833         optional => 1,
1834     },
1835 };
1836
1837 sub parse_smbios1 {
1838     my ($data) = @_;
1839
1840     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1841     warn $@ if $@;
1842     return $res;
1843 }
1844
1845 sub print_smbios1 {
1846     my ($smbios1) = @_;
1847     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1848 }
1849
1850 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1851
1852 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1853 sub verify_bootdisk {
1854     my ($value, $noerr) = @_;
1855
1856     return $value if is_valid_drivename($value);
1857
1858     return undef if $noerr;
1859
1860     die "invalid boot disk '$value'\n";
1861 }
1862
1863 sub parse_watchdog {
1864     my ($value) = @_;
1865
1866     return undef if !$value;
1867
1868     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1869     warn $@ if $@;
1870     return $res;
1871 }
1872
1873 sub parse_usb_device {
1874     my ($value) = @_;
1875
1876     return undef if !$value;
1877
1878     my $res = {};
1879     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1880         $res->{vendorid} = $2;
1881         $res->{productid} = $4;
1882     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1883         $res->{hostbus} = $1;
1884         $res->{hostport} = $2;
1885     } elsif ($value =~ m/^spice$/i) {
1886         $res->{spice} = 1;
1887     } else {
1888         return undef;
1889     }
1890
1891     return $res;
1892 }
1893
1894 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1895 sub verify_usb_device {
1896     my ($value, $noerr) = @_;
1897
1898     return $value if parse_usb_device($value);
1899
1900     return undef if $noerr;
1901
1902     die "unable to parse usb device\n";
1903 }
1904
1905 # add JSON properties for create and set function
1906 sub json_config_properties {
1907     my $prop = shift;
1908
1909     foreach my $opt (keys %$confdesc) {
1910         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1911         $prop->{$opt} = $confdesc->{$opt};
1912     }
1913
1914     return $prop;
1915 }
1916
1917 sub check_type {
1918     my ($key, $value) = @_;
1919
1920     die "unknown setting '$key'\n" if !$confdesc->{$key};
1921
1922     my $type = $confdesc->{$key}->{type};
1923
1924     if (!defined($value)) {
1925         die "got undefined value\n";
1926     }
1927
1928     if ($value =~ m/[\n\r]/) {
1929         die "property contains a line feed\n";
1930     }
1931
1932     if ($type eq 'boolean') {
1933         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1934         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1935         die "type check ('boolean') failed - got '$value'\n";
1936     } elsif ($type eq 'integer') {
1937         return int($1) if $value =~ m/^(\d+)$/;
1938         die "type check ('integer') failed - got '$value'\n";
1939     } elsif ($type eq 'number') {
1940         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1941         die "type check ('number') failed - got '$value'\n";
1942     } elsif ($type eq 'string') {
1943         if (my $fmt = $confdesc->{$key}->{format}) {
1944             if ($fmt eq 'pve-qm-drive') {
1945                 # special case - we need to pass $key to parse_drive()
1946                 my $drive = parse_drive($key, $value);
1947                 return $value if $drive;
1948                 die "unable to parse drive options\n";
1949             }
1950             PVE::JSONSchema::check_format($fmt, $value);
1951             return $value;
1952         }
1953         $value =~ s/^\"(.*)\"$/$1/;
1954         return $value;
1955     } else {
1956         die "internal error"
1957     }
1958 }
1959
1960 sub check_iommu_support{
1961     #fixme : need to check IOMMU support
1962     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1963
1964     my $iommu=1;
1965     return $iommu;
1966
1967 }
1968
1969 sub touch_config {
1970     my ($vmid) = @_;
1971
1972     my $conf = PVE::QemuConfig->config_file($vmid);
1973     utime undef, undef, $conf;
1974 }
1975
1976 sub destroy_vm {
1977     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1978
1979     my $conffile = PVE::QemuConfig->config_file($vmid);
1980
1981     my $conf = PVE::QemuConfig->load_config($vmid);
1982
1983     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1984
1985     # only remove disks owned by this VM
1986     foreach_drive($conf, sub {
1987         my ($ds, $drive) = @_;
1988
1989         return if drive_is_cdrom($drive);
1990
1991         my $volid = $drive->{file};
1992
1993         return if !$volid || $volid =~ m|^/|;
1994
1995         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1996         return if !$path || !$owner || ($owner != $vmid);
1997
1998         PVE::Storage::vdisk_free($storecfg, $volid);
1999     });
2000
2001     if ($keep_empty_config) {
2002         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2003     } else {
2004         unlink $conffile;
2005     }
2006
2007     # also remove unused disk
2008     eval {
2009         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2010
2011         eval {
2012             PVE::Storage::foreach_volid($dl, sub {
2013                 my ($volid, $sid, $volname, $d) = @_;
2014                 PVE::Storage::vdisk_free($storecfg, $volid);
2015             });
2016         };
2017         warn $@ if $@;
2018
2019     };
2020     warn $@ if $@;
2021 }
2022
2023 sub parse_vm_config {
2024     my ($filename, $raw) = @_;
2025
2026     return undef if !defined($raw);
2027
2028     my $res = {
2029         digest => Digest::SHA::sha1_hex($raw),
2030         snapshots => {},
2031         pending => {},
2032     };
2033
2034     $filename =~ m|/qemu-server/(\d+)\.conf$|
2035         || die "got strange filename '$filename'";
2036
2037     my $vmid = $1;
2038
2039     my $conf = $res;
2040     my $descr;
2041     my $section = '';
2042
2043     my @lines = split(/\n/, $raw);
2044     foreach my $line (@lines) {
2045         next if $line =~ m/^\s*$/;
2046
2047         if ($line =~ m/^\[PENDING\]\s*$/i) {
2048             $section = 'pending';
2049             if (defined($descr)) {
2050                 $descr =~ s/\s+$//;
2051                 $conf->{description} = $descr;
2052             }
2053             $descr = undef;
2054             $conf = $res->{$section} = {};
2055             next;
2056
2057         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2058             $section = $1;
2059             if (defined($descr)) {
2060                 $descr =~ s/\s+$//;
2061                 $conf->{description} = $descr;
2062             }
2063             $descr = undef;
2064             $conf = $res->{snapshots}->{$section} = {};
2065             next;
2066         }
2067
2068         if ($line =~ m/^\#(.*)\s*$/) {
2069             $descr = '' if !defined($descr);
2070             $descr .= PVE::Tools::decode_text($1) . "\n";
2071             next;
2072         }
2073
2074         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2075             $descr = '' if !defined($descr);
2076             $descr .= PVE::Tools::decode_text($2);
2077         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2078             $conf->{snapstate} = $1;
2079         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2080             my $key = $1;
2081             my $value = $2;
2082             $conf->{$key} = $value;
2083         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2084             my $value = $1;
2085             if ($section eq 'pending') {
2086                 $conf->{delete} = $value; # we parse this later
2087             } else {
2088                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2089             }
2090         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2091             my $key = $1;
2092             my $value = $2;
2093             eval { $value = check_type($key, $value); };
2094             if ($@) {
2095                 warn "vm $vmid - unable to parse value of '$key' - $@";
2096             } else {
2097                 my $fmt = $confdesc->{$key}->{format};
2098                 if ($fmt && $fmt eq 'pve-qm-drive') {
2099                     my $v = parse_drive($key, $value);
2100                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2101                         $v->{file} = $volid;
2102                         $value = print_drive($vmid, $v);
2103                     } else {
2104                         warn "vm $vmid - unable to parse value of '$key'\n";
2105                         next;
2106                     }
2107                 }
2108
2109                 if ($key eq 'cdrom') {
2110                     $conf->{ide2} = $value;
2111                 } else {
2112                     $conf->{$key} = $value;
2113                 }
2114             }
2115         }
2116     }
2117
2118     if (defined($descr)) {
2119         $descr =~ s/\s+$//;
2120         $conf->{description} = $descr;
2121     }
2122     delete $res->{snapstate}; # just to be sure
2123
2124     return $res;
2125 }
2126
2127 sub write_vm_config {
2128     my ($filename, $conf) = @_;
2129
2130     delete $conf->{snapstate}; # just to be sure
2131
2132     if ($conf->{cdrom}) {
2133         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2134         $conf->{ide2} = $conf->{cdrom};
2135         delete $conf->{cdrom};
2136     }
2137
2138     # we do not use 'smp' any longer
2139     if ($conf->{sockets}) {
2140         delete $conf->{smp};
2141     } elsif ($conf->{smp}) {
2142         $conf->{sockets} = $conf->{smp};
2143         delete $conf->{cores};
2144         delete $conf->{smp};
2145     }
2146
2147     my $used_volids = {};
2148
2149     my $cleanup_config = sub {
2150         my ($cref, $pending, $snapname) = @_;
2151
2152         foreach my $key (keys %$cref) {
2153             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2154                 $key eq 'snapstate' || $key eq 'pending';
2155             my $value = $cref->{$key};
2156             if ($key eq 'delete') {
2157                 die "propertry 'delete' is only allowed in [PENDING]\n"
2158                     if !$pending;
2159                 # fixme: check syntax?
2160                 next;
2161             }
2162             eval { $value = check_type($key, $value); };
2163             die "unable to parse value of '$key' - $@" if $@;
2164
2165             $cref->{$key} = $value;
2166
2167             if (!$snapname && is_valid_drivename($key)) {
2168                 my $drive = parse_drive($key, $value);
2169                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2170             }
2171         }
2172     };
2173
2174     &$cleanup_config($conf);
2175
2176     &$cleanup_config($conf->{pending}, 1);
2177
2178     foreach my $snapname (keys %{$conf->{snapshots}}) {
2179         die "internal error" if $snapname eq 'pending';
2180         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2181     }
2182
2183     # remove 'unusedX' settings if we re-add a volume
2184     foreach my $key (keys %$conf) {
2185         my $value = $conf->{$key};
2186         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2187             delete $conf->{$key};
2188         }
2189     }
2190
2191     my $generate_raw_config = sub {
2192         my ($conf, $pending) = @_;
2193
2194         my $raw = '';
2195
2196         # add description as comment to top of file
2197         if (defined(my $descr = $conf->{description})) {
2198             if ($descr) {
2199                 foreach my $cl (split(/\n/, $descr)) {
2200                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2201                 }
2202             } else {
2203                 $raw .= "#\n" if $pending;
2204             }
2205         }
2206
2207         foreach my $key (sort keys %$conf) {
2208             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2209             $raw .= "$key: $conf->{$key}\n";
2210         }
2211         return $raw;
2212     };
2213
2214     my $raw = &$generate_raw_config($conf);
2215
2216     if (scalar(keys %{$conf->{pending}})){
2217         $raw .= "\n[PENDING]\n";
2218         $raw .= &$generate_raw_config($conf->{pending}, 1);
2219     }
2220
2221     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2222         $raw .= "\n[$snapname]\n";
2223         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2224     }
2225
2226     return $raw;
2227 }
2228
2229 sub load_defaults {
2230
2231     my $res = {};
2232
2233     # we use static defaults from our JSON schema configuration
2234     foreach my $key (keys %$confdesc) {
2235         if (defined(my $default = $confdesc->{$key}->{default})) {
2236             $res->{$key} = $default;
2237         }
2238     }
2239
2240     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2241     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2242
2243     return $res;
2244 }
2245
2246 sub config_list {
2247     my $vmlist = PVE::Cluster::get_vmlist();
2248     my $res = {};
2249     return $res if !$vmlist || !$vmlist->{ids};
2250     my $ids = $vmlist->{ids};
2251
2252     foreach my $vmid (keys %$ids) {
2253         my $d = $ids->{$vmid};
2254         next if !$d->{node} || $d->{node} ne $nodename;
2255         next if !$d->{type} || $d->{type} ne 'qemu';
2256         $res->{$vmid}->{exists} = 1;
2257     }
2258     return $res;
2259 }
2260
2261 # test if VM uses local resources (to prevent migration)
2262 sub check_local_resources {
2263     my ($conf, $noerr) = @_;
2264
2265     my $loc_res = 0;
2266
2267     $loc_res = 1 if $conf->{hostusb}; # old syntax
2268     $loc_res = 1 if $conf->{hostpci}; # old syntax
2269
2270     foreach my $k (keys %$conf) {
2271         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2272         # sockets are safe: they will recreated be on the target side post-migrate
2273         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2274         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2275     }
2276
2277     die "VM uses local resources\n" if $loc_res && !$noerr;
2278
2279     return $loc_res;
2280 }
2281
2282 # check if used storages are available on all nodes (use by migrate)
2283 sub check_storage_availability {
2284     my ($storecfg, $conf, $node) = @_;
2285
2286     foreach_drive($conf, sub {
2287         my ($ds, $drive) = @_;
2288
2289         my $volid = $drive->{file};
2290         return if !$volid;
2291
2292         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2293         return if !$sid;
2294
2295         # check if storage is available on both nodes
2296         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2297         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2298    });
2299 }
2300
2301 # list nodes where all VM images are available (used by has_feature API)
2302 sub shared_nodes {
2303     my ($conf, $storecfg) = @_;
2304
2305     my $nodelist = PVE::Cluster::get_nodelist();
2306     my $nodehash = { map { $_ => 1 } @$nodelist };
2307     my $nodename = PVE::INotify::nodename();
2308
2309     foreach_drive($conf, sub {
2310         my ($ds, $drive) = @_;
2311
2312         my $volid = $drive->{file};
2313         return if !$volid;
2314
2315         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2316         if ($storeid) {
2317             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2318             if ($scfg->{disable}) {
2319                 $nodehash = {};
2320             } elsif (my $avail = $scfg->{nodes}) {
2321                 foreach my $node (keys %$nodehash) {
2322                     delete $nodehash->{$node} if !$avail->{$node};
2323                 }
2324             } elsif (!$scfg->{shared}) {
2325                 foreach my $node (keys %$nodehash) {
2326                     delete $nodehash->{$node} if $node ne $nodename
2327                 }
2328             }
2329         }
2330     });
2331
2332     return $nodehash
2333 }
2334
2335 sub check_cmdline {
2336     my ($pidfile, $pid) = @_;
2337
2338     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2339     if (defined($fh)) {
2340         my $line = <$fh>;
2341         $fh->close;
2342         return undef if !$line;
2343         my @param = split(/\0/, $line);
2344
2345         my $cmd = $param[0];
2346         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2347
2348         for (my $i = 0; $i < scalar (@param); $i++) {
2349             my $p = $param[$i];
2350             next if !$p;
2351             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2352                 my $p = $param[$i+1];
2353                 return 1 if $p && ($p eq $pidfile);
2354                 return undef;
2355             }
2356         }
2357     }
2358     return undef;
2359 }
2360
2361 sub check_running {
2362     my ($vmid, $nocheck, $node) = @_;
2363
2364     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2365
2366     die "unable to find configuration file for VM $vmid - no such machine\n"
2367         if !$nocheck && ! -f $filename;
2368
2369     my $pidfile = pidfile_name($vmid);
2370
2371     if (my $fd = IO::File->new("<$pidfile")) {
2372         my $st = stat($fd);
2373         my $line = <$fd>;
2374         close($fd);
2375
2376         my $mtime = $st->mtime;
2377         if ($mtime > time()) {
2378             warn "file '$filename' modified in future\n";
2379         }
2380
2381         if ($line =~ m/^(\d+)$/) {
2382             my $pid = $1;
2383             if (check_cmdline($pidfile, $pid)) {
2384                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2385                     return $pid;
2386                 }
2387             }
2388         }
2389     }
2390
2391     return undef;
2392 }
2393
2394 sub vzlist {
2395
2396     my $vzlist = config_list();
2397
2398     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2399
2400     while (defined(my $de = $fd->read)) {
2401         next if $de !~ m/^(\d+)\.pid$/;
2402         my $vmid = $1;
2403         next if !defined($vzlist->{$vmid});
2404         if (my $pid = check_running($vmid)) {
2405             $vzlist->{$vmid}->{pid} = $pid;
2406         }
2407     }
2408
2409     return $vzlist;
2410 }
2411
2412 sub disksize {
2413     my ($storecfg, $conf) = @_;
2414
2415     my $bootdisk = $conf->{bootdisk};
2416     return undef if !$bootdisk;
2417     return undef if !is_valid_drivename($bootdisk);
2418
2419     return undef if !$conf->{$bootdisk};
2420
2421     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2422     return undef if !defined($drive);
2423
2424     return undef if drive_is_cdrom($drive);
2425
2426     my $volid = $drive->{file};
2427     return undef if !$volid;
2428
2429     return $drive->{size};
2430 }
2431
2432 my $last_proc_pid_stat;
2433
2434 # get VM status information
2435 # This must be fast and should not block ($full == false)
2436 # We only query KVM using QMP if $full == true (this can be slow)
2437 sub vmstatus {
2438     my ($opt_vmid, $full) = @_;
2439
2440     my $res = {};
2441
2442     my $storecfg = PVE::Storage::config();
2443
2444     my $list = vzlist();
2445     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2446
2447     my $cpucount = $cpuinfo->{cpus} || 1;
2448
2449     foreach my $vmid (keys %$list) {
2450         next if $opt_vmid && ($vmid ne $opt_vmid);
2451
2452         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2453         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2454
2455         my $d = {};
2456         $d->{pid} = $list->{$vmid}->{pid};
2457
2458         # fixme: better status?
2459         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2460
2461         my $size = disksize($storecfg, $conf);
2462         if (defined($size)) {
2463             $d->{disk} = 0; # no info available
2464             $d->{maxdisk} = $size;
2465         } else {
2466             $d->{disk} = 0;
2467             $d->{maxdisk} = 0;
2468         }
2469
2470         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2471         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2472         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2473
2474         $d->{name} = $conf->{name} || "VM $vmid";
2475         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2476
2477         if ($conf->{balloon}) {
2478             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2479             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2480         }
2481
2482         $d->{uptime} = 0;
2483         $d->{cpu} = 0;
2484         $d->{mem} = 0;
2485
2486         $d->{netout} = 0;
2487         $d->{netin} = 0;
2488
2489         $d->{diskread} = 0;
2490         $d->{diskwrite} = 0;
2491
2492         $d->{template} = PVE::QemuConfig->is_template($conf);
2493
2494         $res->{$vmid} = $d;
2495     }
2496
2497     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2498     foreach my $dev (keys %$netdev) {
2499         next if $dev !~ m/^tap([1-9]\d*)i/;
2500         my $vmid = $1;
2501         my $d = $res->{$vmid};
2502         next if !$d;
2503
2504         $d->{netout} += $netdev->{$dev}->{receive};
2505         $d->{netin} += $netdev->{$dev}->{transmit};
2506
2507         if ($full) {
2508             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2509             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2510         }
2511
2512     }
2513
2514     my $ctime = gettimeofday;
2515
2516     foreach my $vmid (keys %$list) {
2517
2518         my $d = $res->{$vmid};
2519         my $pid = $d->{pid};
2520         next if !$pid;
2521
2522         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2523         next if !$pstat; # not running
2524
2525         my $used = $pstat->{utime} + $pstat->{stime};
2526
2527         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2528
2529         if ($pstat->{vsize}) {
2530             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2531         }
2532
2533         my $old = $last_proc_pid_stat->{$pid};
2534         if (!$old) {
2535             $last_proc_pid_stat->{$pid} = {
2536                 time => $ctime,
2537                 used => $used,
2538                 cpu => 0,
2539             };
2540             next;
2541         }
2542
2543         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2544
2545         if ($dtime > 1000) {
2546             my $dutime = $used -  $old->{used};
2547
2548             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2549             $last_proc_pid_stat->{$pid} = {
2550                 time => $ctime,
2551                 used => $used,
2552                 cpu => $d->{cpu},
2553             };
2554         } else {
2555             $d->{cpu} = $old->{cpu};
2556         }
2557     }
2558
2559     return $res if !$full;
2560
2561     my $qmpclient = PVE::QMPClient->new();
2562
2563     my $ballooncb = sub {
2564         my ($vmid, $resp) = @_;
2565
2566         my $info = $resp->{'return'};
2567         return if !$info->{max_mem};
2568
2569         my $d = $res->{$vmid};
2570
2571         # use memory assigned to VM
2572         $d->{maxmem} = $info->{max_mem};
2573         $d->{balloon} = $info->{actual};
2574
2575         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2576             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2577             $d->{freemem} = $info->{free_mem};
2578         }
2579
2580         $d->{ballooninfo} = $info;
2581     };
2582
2583     my $blockstatscb = sub {
2584         my ($vmid, $resp) = @_;
2585         my $data = $resp->{'return'} || [];
2586         my $totalrdbytes = 0;
2587         my $totalwrbytes = 0;
2588
2589         for my $blockstat (@$data) {
2590             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2591             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2592
2593             $blockstat->{device} =~ s/drive-//;
2594             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2595         }
2596         $res->{$vmid}->{diskread} = $totalrdbytes;
2597         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2598     };
2599
2600     my $statuscb = sub {
2601         my ($vmid, $resp) = @_;
2602
2603         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2604         # this fails if ballon driver is not loaded, so this must be
2605         # the last commnand (following command are aborted if this fails).
2606         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2607
2608         my $status = 'unknown';
2609         if (!defined($status = $resp->{'return'}->{status})) {
2610             warn "unable to get VM status\n";
2611             return;
2612         }
2613
2614         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2615     };
2616
2617     foreach my $vmid (keys %$list) {
2618         next if $opt_vmid && ($vmid ne $opt_vmid);
2619         next if !$res->{$vmid}->{pid}; # not running
2620         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2621     }
2622
2623     $qmpclient->queue_execute(undef, 1);
2624
2625     foreach my $vmid (keys %$list) {
2626         next if $opt_vmid && ($vmid ne $opt_vmid);
2627         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2628     }
2629
2630     return $res;
2631 }
2632
2633 sub foreach_dimm {
2634     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2635
2636     my $dimm_id = 0;
2637     my $current_size = 1024;
2638     my $dimm_size = 512;
2639     return if $current_size == $memory;
2640
2641     for (my $j = 0; $j < 8; $j++) {
2642         for (my $i = 0; $i < 32; $i++) {
2643             my $name = "dimm${dimm_id}";
2644             $dimm_id++;
2645             my $numanode = $i % $sockets;
2646             $current_size += $dimm_size;
2647             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2648             return  $current_size if $current_size >= $memory;
2649         }
2650         $dimm_size *= 2;
2651     }
2652 }
2653
2654 sub foreach_reverse_dimm {
2655     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2656
2657     my $dimm_id = 253;
2658     my $current_size = 4177920;
2659     my $dimm_size = 65536;
2660     return if $current_size == $memory;
2661
2662     for (my $j = 0; $j < 8; $j++) {
2663         for (my $i = 0; $i < 32; $i++) {
2664             my $name = "dimm${dimm_id}";
2665             $dimm_id--;
2666             my $numanode = $i % $sockets;
2667             $current_size -= $dimm_size;
2668             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2669             return  $current_size if $current_size <= $memory;
2670         }
2671         $dimm_size /= 2;
2672     }
2673 }
2674
2675 sub foreach_drive {
2676     my ($conf, $func) = @_;
2677
2678     foreach my $ds (valid_drive_names()) {
2679         next if !defined($conf->{$ds});
2680
2681         my $drive = parse_drive($ds, $conf->{$ds});
2682         next if !$drive;
2683
2684         &$func($ds, $drive);
2685     }
2686 }
2687
2688 sub foreach_volid {
2689     my ($conf, $func) = @_;
2690
2691     my $volhash = {};
2692
2693     my $test_volid = sub {
2694         my ($volid, $is_cdrom) = @_;
2695
2696         return if !$volid;
2697
2698         $volhash->{$volid} = $is_cdrom || 0;
2699     };
2700
2701     foreach_drive($conf, sub {
2702         my ($ds, $drive) = @_;
2703         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2704     });
2705
2706     foreach my $snapname (keys %{$conf->{snapshots}}) {
2707         my $snap = $conf->{snapshots}->{$snapname};
2708         &$test_volid($snap->{vmstate}, 0);
2709         foreach_drive($snap, sub {
2710             my ($ds, $drive) = @_;
2711             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2712         });
2713     }
2714
2715     foreach my $volid (keys %$volhash) {
2716         &$func($volid, $volhash->{$volid});
2717     }
2718 }
2719
2720 sub vga_conf_has_spice {
2721     my ($vga) = @_;
2722
2723     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2724
2725     return $1 || 1;
2726 }
2727
2728 sub config_to_command {
2729     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2730
2731     my $cmd = [];
2732     my $globalFlags = [];
2733     my $machineFlags = [];
2734     my $rtcFlags = [];
2735     my $cpuFlags = [];
2736     my $devices = [];
2737     my $pciaddr = '';
2738     my $bridges = {};
2739     my $kvmver = kvm_user_version();
2740     my $vernum = 0; # unknown
2741     my $ostype = $conf->{ostype};
2742     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2743         $vernum = $1*1000000+$2*1000;
2744     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2745         $vernum = $1*1000000+$2*1000+$3;
2746     }
2747
2748     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2749
2750     my $have_ovz = -f '/proc/vz/vestat';
2751
2752     my $q35 = machine_type_is_q35($conf);
2753     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2754     my $machine_type = $forcemachine || $conf->{machine};
2755     my $use_old_bios_files = undef;
2756     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2757
2758     my $cpuunits = defined($conf->{cpuunits}) ?
2759             $conf->{cpuunits} : $defaults->{cpuunits};
2760
2761     push @$cmd, '/usr/bin/systemd-run';
2762     push @$cmd, '--scope';
2763     push @$cmd, '--slice', "qemu";
2764     push @$cmd, '--unit', $vmid;
2765     # set KillMode=none, so that systemd don't kill those scopes
2766     # at shutdown (pve-manager service should stop the VMs instead)
2767     push @$cmd, '-p', "KillMode=none";
2768     push @$cmd, '-p', "CPUShares=$cpuunits";
2769     if ($conf->{cpulimit}) {
2770         my $cpulimit = int($conf->{cpulimit} * 100);
2771         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2772     }
2773
2774     push @$cmd, '/usr/bin/kvm';
2775
2776     push @$cmd, '-id', $vmid;
2777
2778     my $use_virtio = 0;
2779
2780     my $qmpsocket = qmp_socket($vmid);
2781     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2782     push @$cmd, '-mon', "chardev=qmp,mode=control";
2783
2784
2785     push @$cmd, '-pidfile' , pidfile_name($vmid);
2786
2787     push @$cmd, '-daemonize';
2788
2789     if ($conf->{smbios1}) {
2790         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2791     }
2792
2793     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2794         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2795         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2796         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2797         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2798         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2799         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2800     }
2801
2802     if ($q35) {
2803         # the q35 chipset support native usb2, so we enable usb controller
2804         # by default for this machine type
2805         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2806     } else {
2807         $pciaddr = print_pci_addr("piix3", $bridges);
2808         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2809
2810         my $use_usb2 = 0;
2811         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2812             next if !$conf->{"usb$i"};
2813             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2814             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2815             $use_usb2 = 1;
2816         }
2817         # include usb device config
2818         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2819     }
2820
2821     # add usb3 controller if needed
2822
2823     my $use_usb3 = 0;
2824     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2825         next if !$conf->{"usb$i"};
2826         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2827         next if !$d || !$d->{usb3};
2828         $use_usb3 = 1;
2829     }
2830
2831     $pciaddr = print_pci_addr("xhci", $bridges);
2832     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2833
2834     my $vga = $conf->{vga};
2835
2836     my $qxlnum = vga_conf_has_spice($vga);
2837     $vga = 'qxl' if $qxlnum;
2838
2839     if (!$vga) {
2840         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2841                                 $conf->{ostype} eq 'win7' ||
2842                                 $conf->{ostype} eq 'w2k8')) {
2843             $vga = 'std';
2844         } else {
2845             $vga = 'cirrus';
2846         }
2847     }
2848
2849     # enable absolute mouse coordinates (needed by vnc)
2850     my $tablet;
2851     if (defined($conf->{tablet})) {
2852         $tablet = $conf->{tablet};
2853     } else {
2854         $tablet = $defaults->{tablet};
2855         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2856         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2857     }
2858
2859     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2860
2861     my $kvm_off = 0;
2862     # host pci devices
2863     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2864         my $d = parse_hostpci($conf->{"hostpci$i"});
2865         next if !$d;
2866
2867         my $pcie = $d->{pcie};
2868         if($pcie){
2869             die "q35 machine model is not enabled" if !$q35;
2870             $pciaddr = print_pcie_addr("hostpci$i");
2871         }else{
2872             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2873         }
2874
2875         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2876         my $xvga = '';
2877         if ($d->{'x-vga'}) {
2878             $xvga = ',x-vga=on';
2879             $kvm_off = 1;
2880             $vga = 'none';
2881             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2882                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2883             }
2884             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2885                 $xvga = "";
2886             }
2887         }
2888         my $pcidevices = $d->{pciid};
2889         my $multifunction = 1 if @$pcidevices > 1;
2890
2891         my $j=0;
2892         foreach my $pcidevice (@$pcidevices) {
2893
2894             my $id = "hostpci$i";
2895             $id .= ".$j" if $multifunction;
2896             my $addr = $pciaddr;
2897             $addr .= ".$j" if $multifunction;
2898             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2899
2900             if($j == 0){
2901                 $devicestr .= "$rombar$xvga";
2902                 $devicestr .= ",multifunction=on" if $multifunction;
2903             }
2904
2905             push @$devices, '-device', $devicestr;
2906             $j++;
2907         }
2908     }
2909
2910     # usb devices
2911     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2912         next if !$conf->{"usb$i"};
2913         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2914         next if !$d;
2915
2916         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2917         my $usbbus = '';
2918         if (defined($d->{usb3}) && $d->{usb3}) {
2919             $usbbus = ',bus=xhci.0';
2920         }
2921
2922         if (defined($d->{host})) {
2923             $d = parse_usb_device($d->{host});
2924             if (defined($d->{vendorid}) && defined($d->{productid})) {
2925                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2926             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2927                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2928             } elsif (defined($d->{spice}) && $d->{spice}) {
2929                 # usb redir support for spice, currently no usb3
2930                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2931                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2932             }
2933         }
2934     }
2935
2936     # serial devices
2937     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2938         if (my $path = $conf->{"serial$i"}) {
2939             if ($path eq 'socket') {
2940                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2941                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2942                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2943             } else {
2944                 die "no such serial device\n" if ! -c $path;
2945                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2946                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2947             }
2948         }
2949     }
2950
2951     # parallel devices
2952     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2953         if (my $path = $conf->{"parallel$i"}) {
2954             die "no such parallel device\n" if ! -c $path;
2955             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2956             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2957             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2958         }
2959     }
2960
2961     my $vmname = $conf->{name} || "vm$vmid";
2962
2963     push @$cmd, '-name', $vmname;
2964
2965     my $sockets = 1;
2966     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2967     $sockets = $conf->{sockets} if  $conf->{sockets};
2968
2969     my $cores = $conf->{cores} || 1;
2970
2971     my $maxcpus = $sockets * $cores;
2972
2973     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2974
2975     my $allowed_vcpus = $cpuinfo->{cpus};
2976
2977     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2978         if ($allowed_vcpus < $maxcpus);
2979
2980     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2981
2982     push @$cmd, '-nodefaults';
2983
2984     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2985
2986     my $bootindex_hash = {};
2987     my $i = 1;
2988     foreach my $o (split(//, $bootorder)) {
2989         $bootindex_hash->{$o} = $i*100;
2990         $i++;
2991     }
2992
2993     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2994
2995     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2996
2997     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2998
2999     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3000
3001     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3002         my $socket = vnc_socket($vmid);
3003         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3004     } else {
3005         push @$cmd, '-nographic';
3006     }
3007
3008     # time drift fix
3009     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3010
3011     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3012     my $useLocaltime = $conf->{localtime};
3013
3014     if ($ostype) {
3015         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3016
3017         if ($ostype =~ m/^w/) { # windows
3018             $useLocaltime = 1 if !defined($conf->{localtime});
3019
3020             # use time drift fix when acpi is enabled
3021             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3022                 $tdf = 1 if !defined($conf->{tdf});
3023             }
3024         }
3025
3026         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3027             $ostype eq 'wvista') {
3028             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3029             push @$cmd, '-no-hpet';
3030             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3031                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3032                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3033                 push @$cpuFlags , 'hv_time' if !$nokvm;
3034
3035             } else {
3036                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3037             }
3038         }
3039
3040         if ($ostype eq 'win7' || $ostype eq 'win8') {
3041             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3042         }
3043     }
3044
3045     push @$rtcFlags, 'driftfix=slew' if $tdf;
3046
3047     if ($nokvm) {
3048         push @$machineFlags, 'accel=tcg';
3049     } else {
3050         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3051     }
3052
3053     if ($machine_type) {
3054         push @$machineFlags, "type=${machine_type}";
3055     }
3056
3057     if ($conf->{startdate}) {
3058         push @$rtcFlags, "base=$conf->{startdate}";
3059     } elsif ($useLocaltime) {
3060         push @$rtcFlags, 'base=localtime';
3061     }
3062
3063     my $cpu = $nokvm ? "qemu64" : "kvm64";
3064     if (my $cputype = $conf->{cpu}) {
3065         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3066             or die "Cannot parse cpu description: $cputype\n";
3067         $cpu = $cpuconf->{cputype};
3068         $kvm_off = 1 if $cpuconf->{hidden};
3069     }
3070
3071     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3072
3073     push @$cpuFlags , '-x2apic'
3074         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3075
3076     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3077
3078     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3079
3080     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3081
3082         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3083         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3084     }
3085
3086     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3087
3088     push @$cpuFlags, 'kvm=off' if $kvm_off;
3089
3090     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3091         die "internal error"; # should not happen
3092
3093     push @$cpuFlags, "vendor=${cpu_vendor}"
3094         if $cpu_vendor ne 'default';
3095
3096     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3097
3098     push @$cmd, '-cpu', $cpu;
3099
3100     my $memory = $conf->{memory} || $defaults->{memory};
3101     my $static_memory = 0;
3102     my $dimm_memory = 0;
3103
3104     if ($hotplug_features->{memory}) {
3105         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3106         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3107         $static_memory = $STATICMEM;
3108         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3109         $dimm_memory = $memory - $static_memory;
3110         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3111
3112     } else {
3113
3114         $static_memory = $memory;
3115         push @$cmd, '-m', $static_memory;
3116     }
3117
3118     if ($conf->{numa}) {
3119
3120         my $numa_totalmemory = undef;
3121         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3122             next if !$conf->{"numa$i"};
3123             my $numa = parse_numa($conf->{"numa$i"});
3124             next if !$numa;
3125             # memory
3126             die "missing numa node$i memory value\n" if !$numa->{memory};
3127             my $numa_memory = $numa->{memory};
3128             $numa_totalmemory += $numa_memory;
3129             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3130
3131             # cpus
3132             my $cpulists = $numa->{cpus};
3133             die "missing numa node$i cpus\n" if !defined($cpulists);
3134             my $cpus = join(',', map {
3135                 my ($start, $end) = @$_;
3136                 defined($end) ? "$start-$end" : $start
3137             } @$cpulists);
3138
3139             # hostnodes
3140             my $hostnodelists = $numa->{hostnodes};
3141             if (defined($hostnodelists)) {
3142                 my $hostnodes;
3143                 foreach my $hostnoderange (@$hostnodelists) {
3144                     my ($start, $end) = @$hostnoderange;
3145                     $hostnodes .= ',' if $hostnodes;
3146                     $hostnodes .= $start;
3147                     $hostnodes .= "-$end" if defined($end);
3148                     $end //= $start;
3149                     for (my $i = $start; $i <= $end; ++$i ) {
3150                         die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3151                     }
3152                 }
3153
3154                 # policy
3155                 my $policy = $numa->{policy};
3156                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3157                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3158             }
3159
3160             push @$cmd, '-object', $numa_object;
3161             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3162         }
3163
3164         die "total memory for NUMA nodes must be equal to vm static memory\n"
3165             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3166
3167         #if no custom tology, we split memory and cores across numa nodes
3168         if(!$numa_totalmemory) {
3169
3170             my $numa_memory = ($static_memory / $sockets) . "M";
3171
3172             for (my $i = 0; $i < $sockets; $i++)  {
3173
3174                 my $cpustart = ($cores * $i);
3175                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3176                 my $cpus = $cpustart;
3177                 $cpus .= "-$cpuend" if $cpuend;
3178
3179                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3180                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3181             }
3182         }
3183     }
3184
3185     if ($hotplug_features->{memory}) {
3186         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3187             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3188             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3189             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3190
3191             #if dimm_memory is not aligned to dimm map
3192             if($current_size > $memory) {
3193                  $conf->{memory} = $current_size;
3194                  PVE::QemuConfig->write_config($vmid, $conf);
3195             }
3196         });
3197     }
3198
3199     push @$cmd, '-S' if $conf->{freeze};
3200
3201     # set keyboard layout
3202     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3203     push @$cmd, '-k', $kb if $kb;
3204
3205     # enable sound
3206     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3207     #push @$cmd, '-soundhw', 'es1370';
3208     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3209
3210     if($conf->{agent}) {
3211         my $qgasocket = qmp_socket($vmid, 1);
3212         my $pciaddr = print_pci_addr("qga0", $bridges);
3213         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3214         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3215         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3216     }
3217
3218     my $spice_port;
3219
3220     if ($qxlnum) {
3221         if ($qxlnum > 1) {
3222             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3223                 for(my $i = 1; $i < $qxlnum; $i++){
3224                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3225                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3226                 }
3227             } else {
3228                 # assume other OS works like Linux
3229                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3230                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3231             }
3232         }
3233
3234         my $pciaddr = print_pci_addr("spice", $bridges);
3235
3236         my $nodename = PVE::INotify::nodename();
3237         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3238         $spice_port = PVE::Tools::next_spice_port($pfamily);
3239
3240         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3241
3242         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3243         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3244         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3245     }
3246
3247     # enable balloon by default, unless explicitly disabled
3248     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3249         $pciaddr = print_pci_addr("balloon0", $bridges);
3250         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3251     }
3252
3253     if ($conf->{watchdog}) {
3254         my $wdopts = parse_watchdog($conf->{watchdog});
3255         $pciaddr = print_pci_addr("watchdog", $bridges);
3256         my $watchdog = $wdopts->{model} || 'i6300esb';
3257         push @$devices, '-device', "$watchdog$pciaddr";
3258         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3259     }
3260
3261     my $vollist = [];
3262     my $scsicontroller = {};
3263     my $ahcicontroller = {};
3264     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3265
3266     # Add iscsi initiator name if available
3267     if (my $initiator = get_initiator_name()) {
3268         push @$devices, '-iscsi', "initiator-name=$initiator";
3269     }
3270
3271     foreach_drive($conf, sub {
3272         my ($ds, $drive) = @_;
3273
3274         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3275             push @$vollist, $drive->{file};
3276         }
3277
3278         $use_virtio = 1 if $ds =~ m/^virtio/;
3279
3280         if (drive_is_cdrom ($drive)) {
3281             if ($bootindex_hash->{d}) {
3282                 $drive->{bootindex} = $bootindex_hash->{d};
3283                 $bootindex_hash->{d} += 1;
3284             }
3285         } else {
3286             if ($bootindex_hash->{c}) {
3287                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3288                 $bootindex_hash->{c} += 1;
3289             }
3290         }
3291
3292         if($drive->{interface} eq 'virtio'){
3293            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3294         }
3295
3296         if ($drive->{interface} eq 'scsi') {
3297
3298             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3299
3300             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3301             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3302
3303             my $iothread = '';
3304             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3305                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3306                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3307             } elsif ($drive->{iothread}) {
3308                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3309             }
3310
3311             my $queues = '';
3312             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3313                 $queues = ",num_queues=$drive->{queues}";
3314             }
3315
3316             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3317             $scsicontroller->{$controller}=1;
3318         }
3319
3320         if ($drive->{interface} eq 'sata') {
3321            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3322            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3323            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3324            $ahcicontroller->{$controller}=1;
3325         }
3326
3327         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3328         push @$devices, '-drive',$drive_cmd;
3329         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3330     });
3331
3332     for (my $i = 0; $i < $MAX_NETS; $i++) {
3333          next if !$conf->{"net$i"};
3334          my $d = parse_net($conf->{"net$i"});
3335          next if !$d;
3336
3337          $use_virtio = 1 if $d->{model} eq 'virtio';
3338
3339          if ($bootindex_hash->{n}) {
3340             $d->{bootindex} = $bootindex_hash->{n};
3341             $bootindex_hash->{n} += 1;
3342          }
3343
3344          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3345          push @$devices, '-netdev', $netdevfull;
3346
3347          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3348          push @$devices, '-device', $netdevicefull;
3349     }
3350
3351     if (!$q35) {
3352         # add pci bridges
3353         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3354            $bridges->{1} = 1;
3355            $bridges->{2} = 1;
3356         }
3357
3358         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3359
3360         while (my ($k, $v) = each %$bridges) {
3361             $pciaddr = print_pci_addr("pci.$k");
3362             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3363         }
3364     }
3365
3366     # add custom args
3367     if ($conf->{args}) {
3368         my $aa = PVE::Tools::split_args($conf->{args});
3369         push @$cmd, @$aa;
3370     }
3371
3372     push @$cmd, @$devices;
3373     push @$cmd, '-rtc', join(',', @$rtcFlags)
3374         if scalar(@$rtcFlags);
3375     push @$cmd, '-machine', join(',', @$machineFlags)
3376         if scalar(@$machineFlags);
3377     push @$cmd, '-global', join(',', @$globalFlags)
3378         if scalar(@$globalFlags);
3379
3380     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3381 }
3382
3383 sub vnc_socket {
3384     my ($vmid) = @_;
3385     return "${var_run_tmpdir}/$vmid.vnc";
3386 }
3387
3388 sub spice_port {
3389     my ($vmid) = @_;
3390
3391     my $res = vm_mon_cmd($vmid, 'query-spice');
3392
3393     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3394 }
3395
3396 sub qmp_socket {
3397     my ($vmid, $qga) = @_;
3398     my $sockettype = $qga ? 'qga' : 'qmp';
3399     return "${var_run_tmpdir}/$vmid.$sockettype";
3400 }
3401
3402 sub pidfile_name {
3403     my ($vmid) = @_;
3404     return "${var_run_tmpdir}/$vmid.pid";
3405 }
3406
3407 sub vm_devices_list {
3408     my ($vmid) = @_;
3409
3410     my $res = vm_mon_cmd($vmid, 'query-pci');
3411     my $devices = {};
3412     foreach my $pcibus (@$res) {
3413         foreach my $device (@{$pcibus->{devices}}) {
3414             next if !$device->{'qdev_id'};
3415             if ($device->{'pci_bridge'}) {
3416                 $devices->{$device->{'qdev_id'}} = 1;
3417                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3418                     next if !$bridge_device->{'qdev_id'};
3419                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3420                     $devices->{$device->{'qdev_id'}}++;
3421                 }
3422             } else {
3423                 $devices->{$device->{'qdev_id'}} = 1;
3424             }
3425         }
3426     }
3427
3428     my $resblock = vm_mon_cmd($vmid, 'query-block');
3429     foreach my $block (@$resblock) {
3430         if($block->{device} =~ m/^drive-(\S+)/){
3431                 $devices->{$1} = 1;
3432         }
3433     }
3434
3435     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3436     foreach my $mice (@$resmice) {
3437         if ($mice->{name} eq 'QEMU HID Tablet') {
3438             $devices->{tablet} = 1;
3439             last;
3440         }
3441     }
3442
3443     return $devices;
3444 }
3445
3446 sub vm_deviceplug {
3447     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3448
3449     my $q35 = machine_type_is_q35($conf);
3450
3451     my $devices_list = vm_devices_list($vmid);
3452     return 1 if defined($devices_list->{$deviceid});
3453
3454     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3455
3456     if ($deviceid eq 'tablet') {
3457
3458         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3459
3460     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3461
3462         qemu_iothread_add($vmid, $deviceid, $device);
3463
3464         qemu_driveadd($storecfg, $vmid, $device);
3465         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3466
3467         qemu_deviceadd($vmid, $devicefull);
3468         eval { qemu_deviceaddverify($vmid, $deviceid); };
3469         if (my $err = $@) {
3470             eval { qemu_drivedel($vmid, $deviceid); };
3471             warn $@ if $@;
3472             die $err;
3473         }
3474
3475     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3476
3477
3478         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3479         my $pciaddr = print_pci_addr($deviceid);
3480         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3481
3482         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3483
3484         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3485             qemu_iothread_add($vmid, $deviceid, $device);
3486             $devicefull .= ",iothread=iothread-$deviceid";
3487         }
3488
3489         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3490             $devicefull .= ",num_queues=$device->{queues}";
3491         }
3492
3493         qemu_deviceadd($vmid, $devicefull);
3494         qemu_deviceaddverify($vmid, $deviceid);
3495
3496     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3497
3498         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3499         qemu_driveadd($storecfg, $vmid, $device);
3500
3501         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3502         eval { qemu_deviceadd($vmid, $devicefull); };
3503         if (my $err = $@) {
3504             eval { qemu_drivedel($vmid, $deviceid); };
3505             warn $@ if $@;
3506             die $err;
3507         }
3508
3509     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3510
3511         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3512
3513         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3514         my $use_old_bios_files = undef;
3515         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3516
3517         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3518         qemu_deviceadd($vmid, $netdevicefull);
3519         eval { qemu_deviceaddverify($vmid, $deviceid); };
3520         if (my $err = $@) {
3521             eval { qemu_netdevdel($vmid, $deviceid); };
3522             warn $@ if $@;
3523             die $err;
3524         }
3525
3526     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3527
3528         my $bridgeid = $2;
3529         my $pciaddr = print_pci_addr($deviceid);
3530         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3531
3532         qemu_deviceadd($vmid, $devicefull);
3533         qemu_deviceaddverify($vmid, $deviceid);
3534
3535     } else {
3536         die "can't hotplug device '$deviceid'\n";
3537     }
3538
3539     return 1;
3540 }
3541
3542 # fixme: this should raise exceptions on error!
3543 sub vm_deviceunplug {
3544     my ($vmid, $conf, $deviceid) = @_;
3545
3546     my $devices_list = vm_devices_list($vmid);
3547     return 1 if !defined($devices_list->{$deviceid});
3548
3549     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3550
3551     if ($deviceid eq 'tablet') {
3552
3553         qemu_devicedel($vmid, $deviceid);
3554
3555     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3556
3557         qemu_devicedel($vmid, $deviceid);
3558         qemu_devicedelverify($vmid, $deviceid);
3559         qemu_drivedel($vmid, $deviceid);
3560         qemu_iothread_del($conf, $vmid, $deviceid);
3561
3562     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3563
3564         qemu_devicedel($vmid, $deviceid);
3565         qemu_devicedelverify($vmid, $deviceid);
3566         qemu_iothread_del($conf, $vmid, $deviceid);
3567
3568     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3569
3570         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3571         my $device = parse_drive($deviceid, $conf->{$deviceid});
3572         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3573
3574         qemu_devicedel($vmid, $deviceid);
3575         qemu_drivedel($vmid, $deviceid);
3576         qemu_deletescsihw($conf, $vmid, $deviceid);
3577
3578     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3579
3580         qemu_devicedel($vmid, $deviceid);
3581         qemu_devicedelverify($vmid, $deviceid);
3582         qemu_netdevdel($vmid, $deviceid);
3583
3584     } else {
3585         die "can't unplug device '$deviceid'\n";
3586     }
3587
3588     return 1;
3589 }
3590
3591 sub qemu_deviceadd {
3592     my ($vmid, $devicefull) = @_;
3593
3594     $devicefull = "driver=".$devicefull;
3595     my %options =  split(/[=,]/, $devicefull);
3596
3597     vm_mon_cmd($vmid, "device_add" , %options);
3598 }
3599
3600 sub qemu_devicedel {
3601     my ($vmid, $deviceid) = @_;
3602
3603     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3604 }
3605
3606 sub qemu_iothread_add {
3607     my($vmid, $deviceid, $device) = @_;
3608
3609     if ($device->{iothread}) {
3610         my $iothreads = vm_iothreads_list($vmid);
3611         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3612     }
3613 }
3614
3615 sub qemu_iothread_del {
3616     my($conf, $vmid, $deviceid) = @_;
3617
3618     my $device = parse_drive($deviceid, $conf->{$deviceid});
3619     if ($device->{iothread}) {
3620         my $iothreads = vm_iothreads_list($vmid);
3621         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3622     }
3623 }
3624
3625 sub qemu_objectadd {
3626     my($vmid, $objectid, $qomtype) = @_;
3627
3628     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3629
3630     return 1;
3631 }
3632
3633 sub qemu_objectdel {
3634     my($vmid, $objectid) = @_;
3635
3636     vm_mon_cmd($vmid, "object-del", id => $objectid);
3637
3638     return 1;
3639 }
3640
3641 sub qemu_driveadd {
3642     my ($storecfg, $vmid, $device) = @_;
3643
3644     my $drive = print_drive_full($storecfg, $vmid, $device);
3645     $drive =~ s/\\/\\\\/g;
3646     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3647
3648     # If the command succeeds qemu prints: "OK"
3649     return 1 if $ret =~ m/OK/s;
3650
3651     die "adding drive failed: $ret\n";
3652 }
3653
3654 sub qemu_drivedel {
3655     my($vmid, $deviceid) = @_;
3656
3657     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3658     $ret =~ s/^\s+//;
3659
3660     return 1 if $ret eq "";
3661
3662     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3663     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3664
3665     die "deleting drive $deviceid failed : $ret\n";
3666 }
3667
3668 sub qemu_deviceaddverify {
3669     my ($vmid, $deviceid) = @_;
3670
3671     for (my $i = 0; $i <= 5; $i++) {
3672          my $devices_list = vm_devices_list($vmid);
3673          return 1 if defined($devices_list->{$deviceid});
3674          sleep 1;
3675     }
3676
3677     die "error on hotplug device '$deviceid'\n";
3678 }
3679
3680
3681 sub qemu_devicedelverify {
3682     my ($vmid, $deviceid) = @_;
3683
3684     # need to verify that the device is correctly removed as device_del
3685     # is async and empty return is not reliable
3686
3687     for (my $i = 0; $i <= 5; $i++) {
3688          my $devices_list = vm_devices_list($vmid);
3689          return 1 if !defined($devices_list->{$deviceid});
3690          sleep 1;
3691     }
3692
3693     die "error on hot-unplugging device '$deviceid'\n";
3694 }
3695
3696 sub qemu_findorcreatescsihw {
3697     my ($storecfg, $conf, $vmid, $device) = @_;
3698
3699     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3700
3701     my $scsihwid="$controller_prefix$controller";
3702     my $devices_list = vm_devices_list($vmid);
3703
3704     if(!defined($devices_list->{$scsihwid})) {
3705         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3706     }
3707
3708     return 1;
3709 }
3710
3711 sub qemu_deletescsihw {
3712     my ($conf, $vmid, $opt) = @_;
3713
3714     my $device = parse_drive($opt, $conf->{$opt});
3715
3716     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3717         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3718         return 1;
3719     }
3720
3721     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3722
3723     my $devices_list = vm_devices_list($vmid);
3724     foreach my $opt (keys %{$devices_list}) {
3725         if (PVE::QemuServer::is_valid_drivename($opt)) {
3726             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3727             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3728                 return 1;
3729             }
3730         }
3731     }
3732
3733     my $scsihwid="scsihw$controller";
3734
3735     vm_deviceunplug($vmid, $conf, $scsihwid);
3736
3737     return 1;
3738 }
3739
3740 sub qemu_add_pci_bridge {
3741     my ($storecfg, $conf, $vmid, $device) = @_;
3742
3743     my $bridges = {};
3744
3745     my $bridgeid;
3746
3747     print_pci_addr($device, $bridges);
3748
3749     while (my ($k, $v) = each %$bridges) {
3750         $bridgeid = $k;
3751     }
3752     return 1 if !defined($bridgeid) || $bridgeid < 1;
3753
3754     my $bridge = "pci.$bridgeid";
3755     my $devices_list = vm_devices_list($vmid);
3756
3757     if (!defined($devices_list->{$bridge})) {
3758         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3759     }
3760
3761     return 1;
3762 }
3763
3764 sub qemu_set_link_status {
3765     my ($vmid, $device, $up) = @_;
3766
3767     vm_mon_cmd($vmid, "set_link", name => $device,
3768                up => $up ? JSON::true : JSON::false);
3769 }
3770
3771 sub qemu_netdevadd {
3772     my ($vmid, $conf, $device, $deviceid) = @_;
3773
3774     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3775     my %options =  split(/[=,]/, $netdev);
3776
3777     vm_mon_cmd($vmid, "netdev_add",  %options);
3778     return 1;
3779 }
3780
3781 sub qemu_netdevdel {
3782     my ($vmid, $deviceid) = @_;
3783
3784     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3785 }
3786
3787 sub qemu_cpu_hotplug {
3788     my ($vmid, $conf, $vcpus) = @_;
3789
3790     my $sockets = 1;
3791     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3792     $sockets = $conf->{sockets} if  $conf->{sockets};
3793     my $cores = $conf->{cores} || 1;
3794     my $maxcpus = $sockets * $cores;
3795
3796     $vcpus = $maxcpus if !$vcpus;
3797
3798     die "you can't add more vcpus than maxcpus\n"
3799         if $vcpus > $maxcpus;
3800
3801     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3802     die "online cpu unplug is not yet possible\n"
3803         if $vcpus < $currentvcpus;
3804
3805     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3806     die "vcpus in running vm is different than configuration\n"
3807         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3808
3809     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3810         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3811     }
3812 }
3813
3814 sub qemu_memory_hotplug {
3815     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3816
3817     return $value if !check_running($vmid);
3818
3819     my $memory = $conf->{memory} || $defaults->{memory};
3820     $value = $defaults->{memory} if !$value;
3821     return $value if $value == $memory;
3822
3823     my $static_memory = $STATICMEM;
3824     my $dimm_memory = $memory - $static_memory;
3825
3826     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3827     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3828
3829
3830     my $sockets = 1;
3831     $sockets = $conf->{sockets} if $conf->{sockets};
3832
3833     if($value > $memory) {
3834
3835         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3836             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3837
3838                 return if $current_size <= $conf->{memory};
3839
3840                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3841                 if (my $err = $@) {
3842                     eval { qemu_objectdel($vmid, "mem-$name"); };
3843                     die $err;
3844                 }
3845
3846                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3847                 if (my $err = $@) {
3848                     eval { qemu_objectdel($vmid, "mem-$name"); };
3849                     die $err;
3850                 }
3851                 #update conf after each succesful module hotplug
3852                 $conf->{memory} = $current_size;
3853                 PVE::QemuConfig->write_config($vmid, $conf);
3854         });
3855
3856     } else {
3857
3858         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3859             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3860
3861                 return if $current_size >= $conf->{memory};
3862                 print "try to unplug memory dimm $name\n";
3863
3864                 my $retry = 0;
3865                 while (1) {
3866                     eval { qemu_devicedel($vmid, $name) };
3867                     sleep 3;
3868                     my $dimm_list = qemu_dimm_list($vmid);
3869                     last if !$dimm_list->{$name};
3870                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3871                     $retry++;
3872                 }
3873
3874                 #update conf after each succesful module unplug
3875                 $conf->{memory} = $current_size;
3876
3877                 eval { qemu_objectdel($vmid, "mem-$name"); };
3878                 PVE::QemuConfig->write_config($vmid, $conf);
3879         });
3880     }
3881 }
3882
3883 sub qemu_dimm_list {
3884     my ($vmid) = @_;
3885
3886     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3887     my $dimms = {};
3888
3889     foreach my $dimm (@$dimmarray) {
3890
3891         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3892         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3893         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3894         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3895         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3896     }
3897     return $dimms;
3898 }
3899
3900 sub qemu_block_set_io_throttle {
3901     my ($vmid, $deviceid,
3902         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3903         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3904
3905     return if !check_running($vmid) ;
3906
3907     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3908         bps => int($bps),
3909         bps_rd => int($bps_rd),
3910         bps_wr => int($bps_wr),
3911         iops => int($iops),
3912         iops_rd => int($iops_rd),
3913         iops_wr => int($iops_wr),
3914         bps_max => int($bps_max),
3915         bps_rd_max => int($bps_rd_max),
3916         bps_wr_max => int($bps_wr_max),
3917         iops_max => int($iops_max),
3918         iops_rd_max => int($iops_rd_max),
3919         iops_wr_max => int($iops_wr_max)
3920     );
3921
3922 }
3923
3924 # old code, only used to shutdown old VM after update
3925 sub __read_avail {
3926     my ($fh, $timeout) = @_;
3927
3928     my $sel = new IO::Select;
3929     $sel->add($fh);
3930
3931     my $res = '';
3932     my $buf;
3933
3934     my @ready;
3935     while (scalar (@ready = $sel->can_read($timeout))) {
3936         my $count;
3937         if ($count = $fh->sysread($buf, 8192)) {
3938             if ($buf =~ /^(.*)\(qemu\) $/s) {
3939                 $res .= $1;
3940                 last;
3941             } else {
3942                 $res .= $buf;
3943             }
3944         } else {
3945             if (!defined($count)) {
3946                 die "$!\n";
3947             }
3948             last;
3949         }
3950     }
3951
3952     die "monitor read timeout\n" if !scalar(@ready);
3953
3954     return $res;
3955 }
3956
3957 # old code, only used to shutdown old VM after update
3958 sub vm_monitor_command {
3959     my ($vmid, $cmdstr, $nocheck) = @_;
3960
3961     my $res;
3962
3963     eval {
3964         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3965
3966         my $sname = "${var_run_tmpdir}/$vmid.mon";
3967
3968         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3969             die "unable to connect to VM $vmid socket - $!\n";
3970
3971         my $timeout = 3;
3972
3973         # hack: migrate sometime blocks the monitor (when migrate_downtime
3974         # is set)
3975         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3976             $timeout = 60*60; # 1 hour
3977         }
3978
3979         # read banner;
3980         my $data = __read_avail($sock, $timeout);
3981
3982         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3983             die "got unexpected qemu monitor banner\n";
3984         }
3985
3986         my $sel = new IO::Select;
3987         $sel->add($sock);
3988
3989         if (!scalar(my @ready = $sel->can_write($timeout))) {
3990             die "monitor write error - timeout";
3991         }
3992
3993         my $fullcmd = "$cmdstr\r";
3994
3995         # syslog('info', "VM $vmid monitor command: $cmdstr");
3996
3997         my $b;
3998         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3999             die "monitor write error - $!";
4000         }
4001
4002         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4003
4004         $timeout = 20;
4005
4006         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4007             $timeout = 60*60; # 1 hour
4008         } elsif ($cmdstr =~ m/^(eject|change)/) {
4009             $timeout = 60; # note: cdrom mount command is slow
4010         }
4011         if ($res = __read_avail($sock, $timeout)) {
4012
4013             my @lines = split("\r?\n", $res);
4014
4015             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4016
4017             $res = join("\n", @lines);
4018             $res .= "\n";
4019         }
4020     };
4021
4022     my $err = $@;
4023
4024     if ($err) {
4025         syslog("err", "VM $vmid monitor command failed - $err");
4026         die $err;
4027     }
4028
4029     return $res;
4030 }
4031
4032 sub qemu_block_resize {
4033     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4034
4035     my $running = check_running($vmid);
4036
4037     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4038
4039     return if !$running;
4040
4041     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4042
4043 }
4044
4045 sub qemu_volume_snapshot {
4046     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4047
4048     my $running = check_running($vmid);
4049
4050     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4051         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4052     } else {
4053         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4054     }
4055 }
4056
4057 sub qemu_volume_snapshot_delete {
4058     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4059
4060     my $running = check_running($vmid);
4061
4062     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4063
4064     return if !$running;
4065
4066     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4067 }
4068
4069 sub set_migration_caps {
4070     my ($vmid) = @_;
4071
4072     my $cap_ref = [];
4073
4074     my $enabled_cap = {
4075         "auto-converge" => 1,
4076         "xbzrle" => 1,
4077         "x-rdma-pin-all" => 0,
4078         "zero-blocks" => 0,
4079         "compress" => 0
4080     };
4081
4082     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4083
4084     for my $supported_capability (@$supported_capabilities) {
4085         push @$cap_ref, {
4086             capability => $supported_capability->{capability},
4087             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4088         };
4089     }
4090
4091     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4092 }
4093
4094 my $fast_plug_option = {
4095     'lock' => 1,
4096     'name' => 1,
4097     'onboot' => 1,
4098     'shares' => 1,
4099     'startup' => 1,
4100     'description' => 1,
4101 };
4102
4103 # hotplug changes in [PENDING]
4104 # $selection hash can be used to only apply specified options, for
4105 # example: { cores => 1 } (only apply changed 'cores')
4106 # $errors ref is used to return error messages
4107 sub vmconfig_hotplug_pending {
4108     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4109
4110     my $defaults = load_defaults();
4111
4112     # commit values which do not have any impact on running VM first
4113     # Note: those option cannot raise errors, we we do not care about
4114     # $selection and always apply them.
4115
4116     my $add_error = sub {
4117         my ($opt, $msg) = @_;
4118         $errors->{$opt} = "hotplug problem - $msg";
4119     };
4120
4121     my $changes = 0;
4122     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4123         if ($fast_plug_option->{$opt}) {
4124             $conf->{$opt} = $conf->{pending}->{$opt};
4125             delete $conf->{pending}->{$opt};
4126             $changes = 1;
4127         }
4128     }
4129
4130     if ($changes) {
4131         PVE::QemuConfig->write_config($vmid, $conf);
4132         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4133     }
4134
4135     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4136
4137     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4138     while (my ($opt, $force) = each %$pending_delete_hash) {
4139         next if $selection && !$selection->{$opt};
4140         eval {
4141             if ($opt eq 'hotplug') {
4142                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4143             } elsif ($opt eq 'tablet') {
4144                 die "skip\n" if !$hotplug_features->{usb};
4145                 if ($defaults->{tablet}) {
4146                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4147                 } else {
4148                     vm_deviceunplug($vmid, $conf, $opt);
4149                 }
4150             } elsif ($opt eq 'vcpus') {
4151                 die "skip\n" if !$hotplug_features->{cpu};
4152                 qemu_cpu_hotplug($vmid, $conf, undef);
4153             } elsif ($opt eq 'balloon') {
4154                 # enable balloon device is not hotpluggable
4155                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4156             } elsif ($fast_plug_option->{$opt}) {
4157                 # do nothing
4158             } elsif ($opt =~ m/^net(\d+)$/) {
4159                 die "skip\n" if !$hotplug_features->{network};
4160                 vm_deviceunplug($vmid, $conf, $opt);
4161             } elsif (is_valid_drivename($opt)) {
4162                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4163                 vm_deviceunplug($vmid, $conf, $opt);
4164                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4165             } elsif ($opt =~ m/^memory$/) {
4166                 die "skip\n" if !$hotplug_features->{memory};
4167                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4168             } elsif ($opt eq 'cpuunits') {
4169                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4170             } elsif ($opt eq 'cpulimit') {
4171                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4172             } else {
4173                 die "skip\n";
4174             }
4175         };
4176         if (my $err = $@) {
4177             &$add_error($opt, $err) if $err ne "skip\n";
4178         } else {
4179             # save new config if hotplug was successful
4180             delete $conf->{$opt};
4181             vmconfig_undelete_pending_option($conf, $opt);
4182             PVE::QemuConfig->write_config($vmid, $conf);
4183             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4184         }
4185     }
4186
4187     foreach my $opt (keys %{$conf->{pending}}) {
4188         next if $selection && !$selection->{$opt};
4189         my $value = $conf->{pending}->{$opt};
4190         eval {
4191             if ($opt eq 'hotplug') {
4192                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4193             } elsif ($opt eq 'tablet') {
4194                 die "skip\n" if !$hotplug_features->{usb};
4195                 if ($value == 1) {
4196                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4197                 } elsif ($value == 0) {
4198                     vm_deviceunplug($vmid, $conf, $opt);
4199                 }
4200             } elsif ($opt eq 'vcpus') {
4201                 die "skip\n" if !$hotplug_features->{cpu};
4202                 qemu_cpu_hotplug($vmid, $conf, $value);
4203             } elsif ($opt eq 'balloon') {
4204                 # enable/disable balloning device is not hotpluggable
4205                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4206                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4207                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4208
4209                 # allow manual ballooning if shares is set to zero
4210                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4211                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4212                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4213                 }
4214             } elsif ($opt =~ m/^net(\d+)$/) {
4215                 # some changes can be done without hotplug
4216                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4217                                     $vmid, $opt, $value);
4218             } elsif (is_valid_drivename($opt)) {
4219                 # some changes can be done without hotplug
4220                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4221                                      $vmid, $opt, $value, 1);
4222             } elsif ($opt =~ m/^memory$/) { #dimms
4223                 die "skip\n" if !$hotplug_features->{memory};
4224                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4225             } elsif ($opt eq 'cpuunits') {
4226                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4227             } elsif ($opt eq 'cpulimit') {
4228                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4229                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4230             } else {
4231                 die "skip\n";  # skip non-hot-pluggable options
4232             }
4233         };
4234         if (my $err = $@) {
4235             &$add_error($opt, $err) if $err ne "skip\n";
4236         } else {
4237             # save new config if hotplug was successful
4238             $conf->{$opt} = $value;
4239             delete $conf->{pending}->{$opt};
4240             PVE::QemuConfig->write_config($vmid, $conf);
4241             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4242         }
4243     }
4244 }
4245
4246 sub try_deallocate_drive {
4247     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4248
4249     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4250         my $volid = $drive->{file};
4251         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4252             my $sid = PVE::Storage::parse_volume_id($volid);
4253             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4254
4255             # check if the disk is really unused
4256             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4257                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4258             PVE::Storage::vdisk_free($storecfg, $volid);
4259             return 1;
4260         } else {
4261             # If vm is not owner of this disk remove from config
4262             return 1;
4263         }
4264     }
4265
4266     return undef;
4267 }
4268
4269 sub vmconfig_delete_or_detach_drive {
4270     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4271
4272     my $drive = parse_drive($opt, $conf->{$opt});
4273
4274     my $rpcenv = PVE::RPCEnvironment::get();
4275     my $authuser = $rpcenv->get_user();
4276
4277     if ($force) {
4278         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4279         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4280     } else {
4281         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4282     }
4283 }
4284
4285 sub vmconfig_apply_pending {
4286     my ($vmid, $conf, $storecfg) = @_;
4287
4288     # cold plug
4289
4290     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4291     while (my ($opt, $force) = each %$pending_delete_hash) {
4292         die "internal error" if $opt =~ m/^unused/;
4293         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4294         if (!defined($conf->{$opt})) {
4295             vmconfig_undelete_pending_option($conf, $opt);
4296             PVE::QemuConfig->write_config($vmid, $conf);
4297         } elsif (is_valid_drivename($opt)) {
4298             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4299             vmconfig_undelete_pending_option($conf, $opt);
4300             delete $conf->{$opt};
4301             PVE::QemuConfig->write_config($vmid, $conf);
4302         } else {
4303             vmconfig_undelete_pending_option($conf, $opt);
4304             delete $conf->{$opt};
4305             PVE::QemuConfig->write_config($vmid, $conf);
4306         }
4307     }
4308
4309     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4310
4311     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4312         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4313
4314         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4315             # skip if nothing changed
4316         } elsif (is_valid_drivename($opt)) {
4317             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4318                 if defined($conf->{$opt});
4319             $conf->{$opt} = $conf->{pending}->{$opt};
4320         } else {
4321             $conf->{$opt} = $conf->{pending}->{$opt};
4322         }
4323
4324         delete $conf->{pending}->{$opt};
4325         PVE::QemuConfig->write_config($vmid, $conf);
4326     }
4327 }
4328
4329 my $safe_num_ne = sub {
4330     my ($a, $b) = @_;
4331
4332     return 0 if !defined($a) && !defined($b);
4333     return 1 if !defined($a);
4334     return 1 if !defined($b);
4335
4336     return $a != $b;
4337 };
4338
4339 my $safe_string_ne = sub {
4340     my ($a, $b) = @_;
4341
4342     return 0 if !defined($a) && !defined($b);
4343     return 1 if !defined($a);
4344     return 1 if !defined($b);
4345
4346     return $a ne $b;
4347 };
4348
4349 sub vmconfig_update_net {
4350     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4351
4352     my $newnet = parse_net($value);
4353
4354     if ($conf->{$opt}) {
4355         my $oldnet = parse_net($conf->{$opt});
4356
4357         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4358             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4359             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4360             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4361
4362             # for non online change, we try to hot-unplug
4363             die "skip\n" if !$hotplug;
4364             vm_deviceunplug($vmid, $conf, $opt);
4365         } else {
4366
4367             die "internal error" if $opt !~ m/net(\d+)/;
4368             my $iface = "tap${vmid}i$1";
4369
4370             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4371                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4372                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4373                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4374                 PVE::Network::tap_unplug($iface);
4375                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4376             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4377                 # Rate can be applied on its own but any change above needs to
4378                 # include the rate in tap_plug since OVS resets everything.
4379                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4380             }
4381
4382             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4383                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4384             }
4385
4386             return 1;
4387         }
4388     }
4389
4390     if ($hotplug) {
4391         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4392     } else {
4393         die "skip\n";
4394     }
4395 }
4396
4397 sub vmconfig_update_disk {
4398     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4399
4400     # fixme: do we need force?
4401
4402     my $drive = parse_drive($opt, $value);
4403
4404     if ($conf->{$opt}) {
4405
4406         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4407
4408             my $media = $drive->{media} || 'disk';
4409             my $oldmedia = $old_drive->{media} || 'disk';
4410             die "unable to change media type\n" if $media ne $oldmedia;
4411
4412             if (!drive_is_cdrom($old_drive)) {
4413
4414                 if ($drive->{file} ne $old_drive->{file}) {
4415
4416                     die "skip\n" if !$hotplug;
4417
4418                     # unplug and register as unused
4419                     vm_deviceunplug($vmid, $conf, $opt);
4420                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4421
4422                 } else {
4423                     # update existing disk
4424
4425                     # skip non hotpluggable value
4426                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4427                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4428                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4429                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4430                         die "skip\n";
4431                     }
4432
4433                     # apply throttle
4434                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4435                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4436                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4437                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4438                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4439                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4440                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4441                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4442                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4443                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4444                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4445                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4446
4447                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4448                                                    ($drive->{mbps} || 0)*1024*1024,
4449                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4450                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4451                                                    $drive->{iops} || 0,
4452                                                    $drive->{iops_rd} || 0,
4453                                                    $drive->{iops_wr} || 0,
4454                                                    ($drive->{mbps_max} || 0)*1024*1024,
4455                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4456                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4457                                                    $drive->{iops_max} || 0,
4458                                                    $drive->{iops_rd_max} || 0,
4459                                                    $drive->{iops_wr_max} || 0);
4460
4461                     }
4462
4463                     return 1;
4464                 }
4465
4466             } else { # cdrom
4467
4468                 if ($drive->{file} eq 'none') {
4469                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4470                 } else {
4471                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4472                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4473                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4474                 }
4475
4476                 return 1;
4477             }
4478         }
4479     }
4480
4481     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4482     # hotplug new disks
4483     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4484     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4485 }
4486
4487 sub vm_start {
4488     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4489         $forcemachine, $spice_ticket) = @_;
4490
4491     PVE::QemuConfig->lock_config($vmid, sub {
4492         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4493
4494         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4495
4496         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4497
4498         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4499
4500         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4501             vmconfig_apply_pending($vmid, $conf, $storecfg);
4502             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4503         }
4504
4505         my $defaults = load_defaults();
4506
4507         # set environment variable useful inside network script
4508         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4509
4510         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4511
4512         my $migrate_port = 0;
4513         my $migrate_uri;
4514         if ($statefile) {
4515             if ($statefile eq 'tcp') {
4516                 my $localip = "localhost";
4517                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4518                 my $nodename = PVE::INotify::nodename();
4519                 if ($datacenterconf->{migration_unsecure}) {
4520                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4521                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4522                 }
4523                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4524                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4525                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4526                 push @$cmd, '-incoming', $migrate_uri;
4527                 push @$cmd, '-S';
4528             } else {
4529                 push @$cmd, '-loadstate', $statefile;
4530             }
4531         } elsif ($paused) {
4532             push @$cmd, '-S';
4533         }
4534
4535         # host pci devices
4536         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4537           my $d = parse_hostpci($conf->{"hostpci$i"});
4538           next if !$d;
4539           my $pcidevices = $d->{pciid};
4540           foreach my $pcidevice (@$pcidevices) {
4541                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4542
4543                 my $info = pci_device_info("0000:$pciid");
4544                 die "IOMMU not present\n" if !check_iommu_support();
4545                 die "no pci device info for device '$pciid'\n" if !$info;
4546                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4547                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4548           }
4549         }
4550
4551         PVE::Storage::activate_volumes($storecfg, $vollist);
4552
4553         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4554             my $cmd = [];
4555             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4556             eval  { run_command($cmd); };
4557         }
4558
4559         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4560                     umask => 0077); };
4561
4562         if (my $err = $@) {
4563             # deactivate volumes if start fails
4564             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4565             die "start failed: $err";
4566         }
4567
4568         print "migration listens on $migrate_uri\n" if $migrate_uri;
4569
4570         if ($statefile && $statefile ne 'tcp')  {
4571             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4572             warn $@ if $@;
4573         }
4574
4575         if ($migratedfrom) {
4576
4577             eval {
4578                 set_migration_caps($vmid);
4579             };
4580             warn $@ if $@;
4581
4582             if ($spice_port) {
4583                 print "spice listens on port $spice_port\n";
4584                 if ($spice_ticket) {
4585                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4586                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4587                 }
4588             }
4589
4590         } else {
4591
4592             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4593                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4594                     if $conf->{balloon};
4595             }
4596
4597             foreach my $opt (keys %$conf) {
4598                 next if $opt !~  m/^net\d+$/;
4599                 my $nicconf = parse_net($conf->{$opt});
4600                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4601             }
4602         }
4603
4604         vm_mon_cmd_nocheck($vmid, 'qom-set',
4605                     path => "machine/peripheral/balloon0",
4606                     property => "guest-stats-polling-interval",
4607                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4608
4609     });
4610 }
4611
4612 sub vm_mon_cmd {
4613     my ($vmid, $execute, %params) = @_;
4614
4615     my $cmd = { execute => $execute, arguments => \%params };
4616     vm_qmp_command($vmid, $cmd);
4617 }
4618
4619 sub vm_mon_cmd_nocheck {
4620     my ($vmid, $execute, %params) = @_;
4621
4622     my $cmd = { execute => $execute, arguments => \%params };
4623     vm_qmp_command($vmid, $cmd, 1);
4624 }
4625
4626 sub vm_qmp_command {
4627     my ($vmid, $cmd, $nocheck) = @_;
4628
4629     my $res;
4630
4631     my $timeout;
4632     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4633         $timeout = $cmd->{arguments}->{timeout};
4634         delete $cmd->{arguments}->{timeout};
4635     }
4636
4637     eval {
4638         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4639         my $sname = qmp_socket($vmid);
4640         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4641             my $qmpclient = PVE::QMPClient->new();
4642
4643             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4644         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4645             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4646                 if scalar(%{$cmd->{arguments}});
4647             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4648         } else {
4649             die "unable to open monitor socket\n";
4650         }
4651     };
4652     if (my $err = $@) {
4653         syslog("err", "VM $vmid qmp command failed - $err");
4654         die $err;
4655     }
4656
4657     return $res;
4658 }
4659
4660 sub vm_human_monitor_command {
4661     my ($vmid, $cmdline) = @_;
4662
4663     my $res;
4664
4665     my $cmd = {
4666         execute => 'human-monitor-command',
4667         arguments => { 'command-line' => $cmdline},
4668     };
4669
4670     return vm_qmp_command($vmid, $cmd);
4671 }
4672
4673 sub vm_commandline {
4674     my ($storecfg, $vmid) = @_;
4675
4676     my $conf = PVE::QemuConfig->load_config($vmid);
4677
4678     my $defaults = load_defaults();
4679
4680     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4681
4682     return join(' ', @$cmd);
4683 }
4684
4685 sub vm_reset {
4686     my ($vmid, $skiplock) = @_;
4687
4688     PVE::QemuConfig->lock_config($vmid, sub {
4689
4690         my $conf = PVE::QemuConfig->load_config($vmid);
4691
4692         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4693
4694         vm_mon_cmd($vmid, "system_reset");
4695     });
4696 }
4697
4698 sub get_vm_volumes {
4699     my ($conf) = @_;
4700
4701     my $vollist = [];
4702     foreach_volid($conf, sub {
4703         my ($volid, $is_cdrom) = @_;
4704
4705         return if $volid =~ m|^/|;
4706
4707         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4708         return if !$sid;
4709
4710         push @$vollist, $volid;
4711     });
4712
4713     return $vollist;
4714 }
4715
4716 sub vm_stop_cleanup {
4717     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4718
4719     eval {
4720
4721         if (!$keepActive) {
4722             my $vollist = get_vm_volumes($conf);
4723             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4724         }
4725
4726         foreach my $ext (qw(mon qmp pid vnc qga)) {
4727             unlink "/var/run/qemu-server/${vmid}.$ext";
4728         }
4729
4730         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4731     };
4732     warn $@ if $@; # avoid errors - just warn
4733 }
4734
4735 # Note: use $nockeck to skip tests if VM configuration file exists.
4736 # We need that when migration VMs to other nodes (files already moved)
4737 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4738 sub vm_stop {
4739     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4740
4741     $force = 1 if !defined($force) && !$shutdown;
4742
4743     if ($migratedfrom){
4744         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4745         kill 15, $pid if $pid;
4746         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4747         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4748         return;
4749     }
4750
4751     PVE::QemuConfig->lock_config($vmid, sub {
4752
4753         my $pid = check_running($vmid, $nocheck);
4754         return if !$pid;
4755
4756         my $conf;
4757         if (!$nocheck) {
4758             $conf = PVE::QemuConfig->load_config($vmid);
4759             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4760             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4761                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4762                 $timeout = $opts->{down} if $opts->{down};
4763             }
4764         }
4765
4766         $timeout = 60 if !defined($timeout);
4767
4768         eval {
4769             if ($shutdown) {
4770                 if (defined($conf) && $conf->{agent}) {
4771                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4772                 } else {
4773                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4774                 }
4775             } else {
4776                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4777             }
4778         };
4779         my $err = $@;
4780
4781         if (!$err) {
4782             my $count = 0;
4783             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4784                 $count++;
4785                 sleep 1;
4786             }
4787
4788             if ($count >= $timeout) {
4789                 if ($force) {
4790                     warn "VM still running - terminating now with SIGTERM\n";
4791                     kill 15, $pid;
4792                 } else {
4793                     die "VM quit/powerdown failed - got timeout\n";
4794                 }
4795             } else {
4796                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4797                 return;
4798             }
4799         } else {
4800             if ($force) {
4801                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4802                 kill 15, $pid;
4803             } else {
4804                 die "VM quit/powerdown failed\n";
4805             }
4806         }
4807
4808         # wait again
4809         $timeout = 10;
4810
4811         my $count = 0;
4812         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4813             $count++;
4814             sleep 1;
4815         }
4816
4817         if ($count >= $timeout) {
4818             warn "VM still running - terminating now with SIGKILL\n";
4819             kill 9, $pid;
4820             sleep 1;
4821         }
4822
4823         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4824    });
4825 }
4826
4827 sub vm_suspend {
4828     my ($vmid, $skiplock) = @_;
4829
4830     PVE::QemuConfig->lock_config($vmid, sub {
4831
4832         my $conf = PVE::QemuConfig->load_config($vmid);
4833
4834         PVE::QemuConfig->check_lock($conf)
4835             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4836
4837         vm_mon_cmd($vmid, "stop");
4838     });
4839 }
4840
4841 sub vm_resume {
4842     my ($vmid, $skiplock, $nocheck) = @_;
4843
4844     PVE::QemuConfig->lock_config($vmid, sub {
4845
4846         if (!$nocheck) {
4847
4848             my $conf = PVE::QemuConfig->load_config($vmid);
4849
4850             PVE::QemuConfig->check_lock($conf)
4851                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4852
4853             vm_mon_cmd($vmid, "cont");
4854
4855         } else {
4856             vm_mon_cmd_nocheck($vmid, "cont");
4857         }
4858     });
4859 }
4860
4861 sub vm_sendkey {
4862     my ($vmid, $skiplock, $key) = @_;
4863
4864     PVE::QemuConfig->lock_config($vmid, sub {
4865
4866         my $conf = PVE::QemuConfig->load_config($vmid);
4867
4868         # there is no qmp command, so we use the human monitor command
4869         vm_human_monitor_command($vmid, "sendkey $key");
4870     });
4871 }
4872
4873 sub vm_destroy {
4874     my ($storecfg, $vmid, $skiplock) = @_;
4875
4876     PVE::QemuConfig->lock_config($vmid, sub {
4877
4878         my $conf = PVE::QemuConfig->load_config($vmid);
4879
4880         if (!check_running($vmid)) {
4881             destroy_vm($storecfg, $vmid, undef, $skiplock);
4882         } else {
4883             die "VM $vmid is running - destroy failed\n";
4884         }
4885     });
4886 }
4887
4888 # pci helpers
4889
4890 sub file_write {
4891     my ($filename, $buf) = @_;
4892
4893     my $fh = IO::File->new($filename, "w");
4894     return undef if !$fh;
4895
4896     my $res = print $fh $buf;
4897
4898     $fh->close();
4899
4900     return $res;
4901 }
4902
4903 sub pci_device_info {
4904     my ($name) = @_;
4905
4906     my $res;
4907
4908     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4909     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4910
4911     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4912     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4913
4914     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4915     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4916
4917     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4918     return undef if !defined($product) || $product !~ s/^0x//;
4919
4920     $res = {
4921         name => $name,
4922         vendor => $vendor,
4923         product => $product,
4924         domain => $domain,
4925         bus => $bus,
4926         slot => $slot,
4927         func => $func,
4928         irq => $irq,
4929         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4930     };
4931
4932     return $res;
4933 }
4934
4935 sub pci_dev_reset {
4936     my ($dev) = @_;
4937
4938     my $name = $dev->{name};
4939
4940     my $fn = "$pcisysfs/devices/$name/reset";
4941
4942     return file_write($fn, "1");
4943 }
4944
4945 sub pci_dev_bind_to_vfio {
4946     my ($dev) = @_;
4947
4948     my $name = $dev->{name};
4949
4950     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4951
4952     if (!-d $vfio_basedir) {
4953         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4954     }
4955     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4956
4957     my $testdir = "$vfio_basedir/$name";
4958     return 1 if -d $testdir;
4959
4960     my $data = "$dev->{vendor} $dev->{product}";
4961     return undef if !file_write("$vfio_basedir/new_id", $data);
4962
4963     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4964     if (!file_write($fn, $name)) {
4965         return undef if -f $fn;
4966     }
4967
4968     $fn = "$vfio_basedir/bind";
4969     if (! -d $testdir) {
4970         return undef if !file_write($fn, $name);
4971     }
4972
4973     return -d $testdir;
4974 }
4975
4976 sub pci_dev_group_bind_to_vfio {
4977     my ($pciid) = @_;
4978
4979     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4980
4981     if (!-d $vfio_basedir) {
4982         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4983     }
4984     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4985
4986     # get IOMMU group devices
4987     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4988       my @devs = grep /^0000:/, readdir($D);
4989     closedir($D);
4990
4991     foreach my $pciid (@devs) {
4992         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4993
4994         # pci bridges, switches or root ports are not supported
4995         # they have a pci_bus subdirectory so skip them
4996         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4997
4998         my $info = pci_device_info($1);
4999         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5000     }
5001
5002     return 1;
5003 }
5004
5005 sub print_pci_addr {
5006     my ($id, $bridges) = @_;
5007
5008     my $res = '';
5009     my $devices = {
5010         piix3 => { bus => 0, addr => 1 },
5011         #addr2 : first videocard
5012         balloon0 => { bus => 0, addr => 3 },
5013         watchdog => { bus => 0, addr => 4 },
5014         scsihw0 => { bus => 0, addr => 5 },
5015         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
5016         scsihw1 => { bus => 0, addr => 6 },
5017         ahci0 => { bus => 0, addr => 7 },
5018         qga0 => { bus => 0, addr => 8 },
5019         spice => { bus => 0, addr => 9 },
5020         virtio0 => { bus => 0, addr => 10 },
5021         virtio1 => { bus => 0, addr => 11 },
5022         virtio2 => { bus => 0, addr => 12 },
5023         virtio3 => { bus => 0, addr => 13 },
5024         virtio4 => { bus => 0, addr => 14 },
5025         virtio5 => { bus => 0, addr => 15 },
5026         hostpci0 => { bus => 0, addr => 16 },
5027         hostpci1 => { bus => 0, addr => 17 },
5028         net0 => { bus => 0, addr => 18 },
5029         net1 => { bus => 0, addr => 19 },
5030         net2 => { bus => 0, addr => 20 },
5031         net3 => { bus => 0, addr => 21 },
5032         net4 => { bus => 0, addr => 22 },
5033         net5 => { bus => 0, addr => 23 },
5034         vga1 => { bus => 0, addr => 24 },
5035         vga2 => { bus => 0, addr => 25 },
5036         vga3 => { bus => 0, addr => 26 },
5037         hostpci2 => { bus => 0, addr => 27 },
5038         hostpci3 => { bus => 0, addr => 28 },
5039         #addr29 : usb-host (pve-usb.cfg)
5040         'pci.1' => { bus => 0, addr => 30 },
5041         'pci.2' => { bus => 0, addr => 31 },
5042         'net6' => { bus => 1, addr => 1 },
5043         'net7' => { bus => 1, addr => 2 },
5044         'net8' => { bus => 1, addr => 3 },
5045         'net9' => { bus => 1, addr => 4 },
5046         'net10' => { bus => 1, addr => 5 },
5047         'net11' => { bus => 1, addr => 6 },
5048         'net12' => { bus => 1, addr => 7 },
5049         'net13' => { bus => 1, addr => 8 },
5050         'net14' => { bus => 1, addr => 9 },
5051         'net15' => { bus => 1, addr => 10 },
5052         'net16' => { bus => 1, addr => 11 },
5053         'net17' => { bus => 1, addr => 12 },
5054         'net18' => { bus => 1, addr => 13 },
5055         'net19' => { bus => 1, addr => 14 },
5056         'net20' => { bus => 1, addr => 15 },
5057         'net21' => { bus => 1, addr => 16 },
5058         'net22' => { bus => 1, addr => 17 },
5059         'net23' => { bus => 1, addr => 18 },
5060         'net24' => { bus => 1, addr => 19 },
5061         'net25' => { bus => 1, addr => 20 },
5062         'net26' => { bus => 1, addr => 21 },
5063         'net27' => { bus => 1, addr => 22 },
5064         'net28' => { bus => 1, addr => 23 },
5065         'net29' => { bus => 1, addr => 24 },
5066         'net30' => { bus => 1, addr => 25 },
5067         'net31' => { bus => 1, addr => 26 },
5068         'xhci' => { bus => 1, addr => 27 },
5069         'virtio6' => { bus => 2, addr => 1 },
5070         'virtio7' => { bus => 2, addr => 2 },
5071         'virtio8' => { bus => 2, addr => 3 },
5072         'virtio9' => { bus => 2, addr => 4 },
5073         'virtio10' => { bus => 2, addr => 5 },
5074         'virtio11' => { bus => 2, addr => 6 },
5075         'virtio12' => { bus => 2, addr => 7 },
5076         'virtio13' => { bus => 2, addr => 8 },
5077         'virtio14' => { bus => 2, addr => 9 },
5078         'virtio15' => { bus => 2, addr => 10 },
5079         'virtioscsi0' => { bus => 3, addr => 1 },
5080         'virtioscsi1' => { bus => 3, addr => 2 },
5081         'virtioscsi2' => { bus => 3, addr => 3 },
5082         'virtioscsi3' => { bus => 3, addr => 4 },
5083         'virtioscsi4' => { bus => 3, addr => 5 },
5084         'virtioscsi5' => { bus => 3, addr => 6 },
5085         'virtioscsi6' => { bus => 3, addr => 7 },
5086         'virtioscsi7' => { bus => 3, addr => 8 },
5087         'virtioscsi8' => { bus => 3, addr => 9 },
5088         'virtioscsi9' => { bus => 3, addr => 10 },
5089         'virtioscsi10' => { bus => 3, addr => 11 },
5090         'virtioscsi11' => { bus => 3, addr => 12 },
5091         'virtioscsi12' => { bus => 3, addr => 13 },
5092         'virtioscsi13' => { bus => 3, addr => 14 },
5093         'virtioscsi14' => { bus => 3, addr => 15 },
5094         'virtioscsi15' => { bus => 3, addr => 16 },
5095         'virtioscsi16' => { bus => 3, addr => 17 },
5096         'virtioscsi17' => { bus => 3, addr => 18 },
5097         'virtioscsi18' => { bus => 3, addr => 19 },
5098         'virtioscsi19' => { bus => 3, addr => 20 },
5099         'virtioscsi20' => { bus => 3, addr => 21 },
5100         'virtioscsi21' => { bus => 3, addr => 22 },
5101         'virtioscsi22' => { bus => 3, addr => 23 },
5102         'virtioscsi23' => { bus => 3, addr => 24 },
5103         'virtioscsi24' => { bus => 3, addr => 25 },
5104         'virtioscsi25' => { bus => 3, addr => 26 },
5105         'virtioscsi26' => { bus => 3, addr => 27 },
5106         'virtioscsi27' => { bus => 3, addr => 28 },
5107         'virtioscsi28' => { bus => 3, addr => 29 },
5108         'virtioscsi29' => { bus => 3, addr => 30 },
5109         'virtioscsi30' => { bus => 3, addr => 31 },
5110
5111     };
5112
5113     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5114            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5115            my $bus = $devices->{$id}->{bus};
5116            $res = ",bus=pci.$bus,addr=$addr";
5117            $bridges->{$bus} = 1 if $bridges;
5118     }
5119     return $res;
5120
5121 }
5122
5123 sub print_pcie_addr {
5124     my ($id) = @_;
5125
5126     my $res = '';
5127     my $devices = {
5128         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5129         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5130         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5131         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5132     };
5133
5134     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5135            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5136            my $bus = $devices->{$id}->{bus};
5137            $res = ",bus=$bus,addr=$addr";
5138     }
5139     return $res;
5140
5141 }
5142
5143 # vzdump restore implementaion
5144
5145 sub tar_archive_read_firstfile {
5146     my $archive = shift;
5147
5148     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5149
5150     # try to detect archive type first
5151     my $pid = open (TMP, "tar tf '$archive'|") ||
5152         die "unable to open file '$archive'\n";
5153     my $firstfile = <TMP>;
5154     kill 15, $pid;
5155     close TMP;
5156
5157     die "ERROR: archive contaions no data\n" if !$firstfile;
5158     chomp $firstfile;
5159
5160     return $firstfile;
5161 }
5162
5163 sub tar_restore_cleanup {
5164     my ($storecfg, $statfile) = @_;
5165
5166     print STDERR "starting cleanup\n";
5167
5168     if (my $fd = IO::File->new($statfile, "r")) {
5169         while (defined(my $line = <$fd>)) {
5170             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5171                 my $volid = $2;
5172                 eval {
5173                     if ($volid =~ m|^/|) {
5174                         unlink $volid || die 'unlink failed\n';
5175                     } else {
5176                         PVE::Storage::vdisk_free($storecfg, $volid);
5177                     }
5178                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5179                 };
5180                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5181             } else {
5182                 print STDERR "unable to parse line in statfile - $line";
5183             }
5184         }
5185         $fd->close();
5186     }
5187 }
5188
5189 sub restore_archive {
5190     my ($archive, $vmid, $user, $opts) = @_;
5191
5192     my $format = $opts->{format};
5193     my $comp;
5194
5195     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5196         $format = 'tar' if !$format;
5197         $comp = 'gzip';
5198     } elsif ($archive =~ m/\.tar$/) {
5199         $format = 'tar' if !$format;
5200     } elsif ($archive =~ m/.tar.lzo$/) {
5201         $format = 'tar' if !$format;
5202         $comp = 'lzop';
5203     } elsif ($archive =~ m/\.vma$/) {
5204         $format = 'vma' if !$format;
5205     } elsif ($archive =~ m/\.vma\.gz$/) {
5206         $format = 'vma' if !$format;
5207         $comp = 'gzip';
5208     } elsif ($archive =~ m/\.vma\.lzo$/) {
5209         $format = 'vma' if !$format;
5210         $comp = 'lzop';
5211     } else {
5212         $format = 'vma' if !$format; # default
5213     }
5214
5215     # try to detect archive format
5216     if ($format eq 'tar') {
5217         return restore_tar_archive($archive, $vmid, $user, $opts);
5218     } else {
5219         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5220     }
5221 }
5222
5223 sub restore_update_config_line {
5224     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5225
5226     return if $line =~ m/^\#qmdump\#/;
5227     return if $line =~ m/^\#vzdump\#/;
5228     return if $line =~ m/^lock:/;
5229     return if $line =~ m/^unused\d+:/;
5230     return if $line =~ m/^parent:/;
5231     return if $line =~ m/^template:/; # restored VM is never a template
5232
5233     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5234         # try to convert old 1.X settings
5235         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5236         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5237             my ($model, $macaddr) = split(/\=/, $devconfig);
5238             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5239             my $net = {
5240                 model => $model,
5241                 bridge => "vmbr$ind",
5242                 macaddr => $macaddr,
5243             };
5244             my $netstr = print_net($net);
5245
5246             print $outfd "net$cookie->{netcount}: $netstr\n";
5247             $cookie->{netcount}++;
5248         }
5249     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5250         my ($id, $netstr) = ($1, $2);
5251         my $net = parse_net($netstr);
5252         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5253         $netstr = print_net($net);
5254         print $outfd "$id: $netstr\n";
5255     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5256         my $virtdev = $1;
5257         my $value = $3;
5258         my $di = parse_drive($virtdev, $value);
5259         if (defined($di->{backup}) && !$di->{backup}) {
5260             print $outfd "#$line";
5261         } elsif ($map->{$virtdev}) {
5262             delete $di->{format}; # format can change on restore
5263             $di->{file} = $map->{$virtdev};
5264             $value = print_drive($vmid, $di);
5265             print $outfd "$virtdev: $value\n";
5266         } else {
5267             print $outfd $line;
5268         }
5269     } else {
5270         print $outfd $line;
5271     }
5272 }
5273
5274 sub scan_volids {
5275     my ($cfg, $vmid) = @_;
5276
5277     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5278
5279     my $volid_hash = {};
5280     foreach my $storeid (keys %$info) {
5281         foreach my $item (@{$info->{$storeid}}) {
5282             next if !($item->{volid} && $item->{size});
5283             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5284             $volid_hash->{$item->{volid}} = $item;
5285         }
5286     }
5287
5288     return $volid_hash;
5289 }
5290
5291 sub is_volume_in_use {
5292     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5293
5294     my $path = PVE::Storage::path($storecfg, $volid);
5295
5296     my $scan_config = sub {
5297         my ($cref, $snapname) = @_;
5298
5299         foreach my $key (keys %$cref) {
5300             my $value = $cref->{$key};
5301             if (is_valid_drivename($key)) {
5302                 next if $skip_drive && $key eq $skip_drive;
5303                 my $drive = parse_drive($key, $value);
5304                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5305                 return 1 if $volid eq $drive->{file};
5306                 if ($drive->{file} =~ m!^/!) {
5307                     return 1 if $drive->{file} eq $path;
5308                 } else {
5309                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5310                     next if !$storeid;
5311                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5312                     next if !$scfg;
5313                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5314                 }
5315             }
5316         }
5317
5318         return 0;
5319     };
5320
5321     return 1 if &$scan_config($conf);
5322
5323     undef $skip_drive;
5324
5325     foreach my $snapname (keys %{$conf->{snapshots}}) {
5326         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5327     }
5328
5329     return 0;
5330 }
5331
5332 sub update_disksize {
5333     my ($vmid, $conf, $volid_hash) = @_;
5334
5335     my $changes;
5336
5337     my $used = {};
5338
5339     # Note: it is allowed to define multiple storages with same path (alias), so
5340     # we need to check both 'volid' and real 'path' (two different volid can point
5341     # to the same path).
5342
5343     my $usedpath = {};
5344
5345     # update size info
5346     foreach my $opt (keys %$conf) {
5347         if (is_valid_drivename($opt)) {
5348             my $drive = parse_drive($opt, $conf->{$opt});
5349             my $volid = $drive->{file};
5350             next if !$volid;
5351
5352             $used->{$volid} = 1;
5353             if ($volid_hash->{$volid} &&
5354                 (my $path = $volid_hash->{$volid}->{path})) {
5355                 $usedpath->{$path} = 1;
5356             }
5357
5358             next if drive_is_cdrom($drive);
5359             next if !$volid_hash->{$volid};
5360
5361             $drive->{size} = $volid_hash->{$volid}->{size};
5362             my $new = print_drive($vmid, $drive);
5363             if ($new ne $conf->{$opt}) {
5364                 $changes = 1;
5365                 $conf->{$opt} = $new;
5366             }
5367         }
5368     }
5369
5370     # remove 'unusedX' entry if volume is used
5371     foreach my $opt (keys %$conf) {
5372         next if $opt !~ m/^unused\d+$/;
5373         my $volid = $conf->{$opt};
5374         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5375         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5376             $changes = 1;
5377             delete $conf->{$opt};
5378         }
5379     }
5380
5381     foreach my $volid (sort keys %$volid_hash) {
5382         next if $volid =~ m/vm-$vmid-state-/;
5383         next if $used->{$volid};
5384         my $path = $volid_hash->{$volid}->{path};
5385         next if !$path; # just to be sure
5386         next if $usedpath->{$path};
5387         $changes = 1;
5388         PVE::QemuConfig->add_unused_volume($conf, $volid);
5389         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5390     }
5391
5392     return $changes;
5393 }
5394
5395 sub rescan {
5396     my ($vmid, $nolock) = @_;
5397
5398     my $cfg = PVE::Storage::config();
5399
5400     my $volid_hash = scan_volids($cfg, $vmid);
5401
5402     my $updatefn =  sub {
5403         my ($vmid) = @_;
5404
5405         my $conf = PVE::QemuConfig->load_config($vmid);
5406
5407         PVE::QemuConfig->check_lock($conf);
5408
5409         my $vm_volids = {};
5410         foreach my $volid (keys %$volid_hash) {
5411             my $info = $volid_hash->{$volid};
5412             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5413         }
5414
5415         my $changes = update_disksize($vmid, $conf, $vm_volids);
5416
5417         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5418     };
5419
5420     if (defined($vmid)) {
5421         if ($nolock) {
5422             &$updatefn($vmid);
5423         } else {
5424             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5425         }
5426     } else {
5427         my $vmlist = config_list();
5428         foreach my $vmid (keys %$vmlist) {
5429             if ($nolock) {
5430                 &$updatefn($vmid);
5431             } else {
5432                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5433             }
5434         }
5435     }
5436 }
5437
5438 sub restore_vma_archive {
5439     my ($archive, $vmid, $user, $opts, $comp) = @_;
5440
5441     my $input = $archive eq '-' ? "<&STDIN" : undef;
5442     my $readfrom = $archive;
5443
5444     my $uncomp = '';
5445     if ($comp) {
5446         $readfrom = '-';
5447         my $qarchive = PVE::Tools::shellquote($archive);
5448         if ($comp eq 'gzip') {
5449             $uncomp = "zcat $qarchive|";
5450         } elsif ($comp eq 'lzop') {
5451             $uncomp = "lzop -d -c $qarchive|";
5452         } else {
5453             die "unknown compression method '$comp'\n";
5454         }
5455
5456     }
5457
5458     my $tmpdir = "/var/tmp/vzdumptmp$$";
5459     rmtree $tmpdir;
5460
5461     # disable interrupts (always do cleanups)
5462     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5463         warn "got interrupt - ignored\n";
5464     };
5465
5466     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5467     POSIX::mkfifo($mapfifo, 0600);
5468     my $fifofh;
5469
5470     my $openfifo = sub {
5471         open($fifofh, '>', $mapfifo) || die $!;
5472     };
5473
5474     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5475
5476     my $oldtimeout;
5477     my $timeout = 5;
5478
5479     my $devinfo = {};
5480
5481     my $rpcenv = PVE::RPCEnvironment::get();
5482
5483     my $conffile = PVE::QemuConfig->config_file($vmid);
5484     my $tmpfn = "$conffile.$$.tmp";
5485
5486     # Note: $oldconf is undef if VM does not exists
5487     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5488     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5489
5490     my $print_devmap = sub {
5491         my $virtdev_hash = {};
5492
5493         my $cfgfn = "$tmpdir/qemu-server.conf";
5494
5495         # we can read the config - that is already extracted
5496         my $fh = IO::File->new($cfgfn, "r") ||
5497             "unable to read qemu-server.conf - $!\n";
5498
5499         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5500         if (-f $fwcfgfn) {
5501             my $pve_firewall_dir = '/etc/pve/firewall';
5502             mkdir $pve_firewall_dir; # make sure the dir exists
5503             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5504         }
5505
5506         while (defined(my $line = <$fh>)) {
5507             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5508                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5509                 die "archive does not contain data for drive '$virtdev'\n"
5510                     if !$devinfo->{$devname};
5511                 if (defined($opts->{storage})) {
5512                     $storeid = $opts->{storage} || 'local';
5513                 } elsif (!$storeid) {
5514                     $storeid = 'local';
5515                 }
5516                 $format = 'raw' if !$format;
5517                 $devinfo->{$devname}->{devname} = $devname;
5518                 $devinfo->{$devname}->{virtdev} = $virtdev;
5519                 $devinfo->{$devname}->{format} = $format;
5520                 $devinfo->{$devname}->{storeid} = $storeid;
5521
5522                 # check permission on storage
5523                 my $pool = $opts->{pool}; # todo: do we need that?
5524                 if ($user ne 'root@pam') {
5525                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5526                 }
5527
5528                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5529             }
5530         }
5531
5532         foreach my $devname (keys %$devinfo) {
5533             die "found no device mapping information for device '$devname'\n"
5534                 if !$devinfo->{$devname}->{virtdev};
5535         }
5536
5537         my $cfg = PVE::Storage::config();
5538
5539         # create empty/temp config
5540         if ($oldconf) {
5541             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5542             foreach_drive($oldconf, sub {
5543                 my ($ds, $drive) = @_;
5544
5545                 return if drive_is_cdrom($drive);
5546
5547                 my $volid = $drive->{file};
5548
5549                 return if !$volid || $volid =~ m|^/|;
5550
5551                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5552                 return if !$path || !$owner || ($owner != $vmid);
5553
5554                 # Note: only delete disk we want to restore
5555                 # other volumes will become unused
5556                 if ($virtdev_hash->{$ds}) {
5557                     PVE::Storage::vdisk_free($cfg, $volid);
5558                 }
5559             });
5560
5561             # delete vmstate files
5562             # since after the restore we have no snapshots anymore
5563             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5564                 my $snap = $oldconf->{snapshots}->{$snapname};
5565                 if ($snap->{vmstate}) {
5566                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5567                     if (my $err = $@) {
5568                         warn $err;
5569                     }
5570                 }
5571             }
5572         }
5573
5574         my $map = {};
5575         foreach my $virtdev (sort keys %$virtdev_hash) {
5576             my $d = $virtdev_hash->{$virtdev};
5577             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5578             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5579
5580             # test if requested format is supported
5581             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5582             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5583             $d->{format} = $defFormat if !$supported;
5584
5585             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5586                                                   $d->{format}, undef, $alloc_size);
5587             print STDERR "new volume ID is '$volid'\n";
5588             $d->{volid} = $volid;
5589             my $path = PVE::Storage::path($cfg, $volid);
5590
5591             PVE::Storage::activate_volumes($cfg,[$volid]);
5592
5593             my $write_zeros = 1;
5594             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5595                 $write_zeros = 0;
5596             }
5597
5598             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5599
5600             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5601             $map->{$virtdev} = $volid;
5602         }
5603
5604         $fh->seek(0, 0) || die "seek failed - $!\n";
5605
5606         my $outfd = new IO::File ($tmpfn, "w") ||
5607             die "unable to write config for VM $vmid\n";
5608
5609         my $cookie = { netcount => 0 };
5610         while (defined(my $line = <$fh>)) {
5611             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5612         }
5613
5614         $fh->close();
5615         $outfd->close();
5616     };
5617
5618     eval {
5619         # enable interrupts
5620         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5621             die "interrupted by signal\n";
5622         };
5623         local $SIG{ALRM} = sub { die "got timeout\n"; };
5624
5625         $oldtimeout = alarm($timeout);
5626
5627         my $parser = sub {
5628             my $line = shift;
5629
5630             print "$line\n";
5631
5632             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5633                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5634                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5635             } elsif ($line =~ m/^CTIME: /) {
5636                 # we correctly received the vma config, so we can disable
5637                 # the timeout now for disk allocation (set to 10 minutes, so
5638                 # that we always timeout if something goes wrong)
5639                 alarm(600);
5640                 &$print_devmap();
5641                 print $fifofh "done\n";
5642                 my $tmp = $oldtimeout || 0;
5643                 $oldtimeout = undef;
5644                 alarm($tmp);
5645                 close($fifofh);
5646             }
5647         };
5648
5649         print "restore vma archive: $cmd\n";
5650         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5651     };
5652     my $err = $@;
5653
5654     alarm($oldtimeout) if $oldtimeout;
5655
5656     my $vollist = [];
5657     foreach my $devname (keys %$devinfo) {
5658         my $volid = $devinfo->{$devname}->{volid};
5659         push @$vollist, $volid if $volid;
5660     }
5661
5662     my $cfg = PVE::Storage::config();
5663     PVE::Storage::deactivate_volumes($cfg, $vollist);
5664
5665     unlink $mapfifo;
5666
5667     if ($err) {
5668         rmtree $tmpdir;
5669         unlink $tmpfn;
5670
5671         foreach my $devname (keys %$devinfo) {
5672             my $volid = $devinfo->{$devname}->{volid};
5673             next if !$volid;
5674             eval {
5675                 if ($volid =~ m|^/|) {
5676                     unlink $volid || die 'unlink failed\n';
5677                 } else {
5678                     PVE::Storage::vdisk_free($cfg, $volid);
5679                 }
5680                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5681             };
5682             print STDERR "unable to cleanup '$volid' - $@" if $@;
5683         }
5684         die $err;
5685     }
5686
5687     rmtree $tmpdir;
5688
5689     rename($tmpfn, $conffile) ||
5690         die "unable to commit configuration file '$conffile'\n";
5691
5692     PVE::Cluster::cfs_update(); # make sure we read new file
5693
5694     eval { rescan($vmid, 1); };
5695     warn $@ if $@;
5696 }
5697
5698 sub restore_tar_archive {
5699     my ($archive, $vmid, $user, $opts) = @_;
5700
5701     if ($archive ne '-') {
5702         my $firstfile = tar_archive_read_firstfile($archive);
5703         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5704             if $firstfile ne 'qemu-server.conf';
5705     }
5706
5707     my $storecfg = PVE::Storage::config();
5708
5709     # destroy existing data - keep empty config
5710     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5711     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5712
5713     my $tocmd = "/usr/lib/qemu-server/qmextract";
5714
5715     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5716     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5717     $tocmd .= ' --prealloc' if $opts->{prealloc};
5718     $tocmd .= ' --info' if $opts->{info};
5719
5720     # tar option "xf" does not autodetect compression when read from STDIN,
5721     # so we pipe to zcat
5722     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5723         PVE::Tools::shellquote("--to-command=$tocmd");
5724
5725     my $tmpdir = "/var/tmp/vzdumptmp$$";
5726     mkpath $tmpdir;
5727
5728     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5729     local $ENV{VZDUMP_VMID} = $vmid;
5730     local $ENV{VZDUMP_USER} = $user;
5731
5732     my $conffile = PVE::QemuConfig->config_file($vmid);
5733     my $tmpfn = "$conffile.$$.tmp";
5734
5735     # disable interrupts (always do cleanups)
5736     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5737         print STDERR "got interrupt - ignored\n";
5738     };
5739
5740     eval {
5741         # enable interrupts
5742         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5743             die "interrupted by signal\n";
5744         };
5745
5746         if ($archive eq '-') {
5747             print "extracting archive from STDIN\n";
5748             run_command($cmd, input => "<&STDIN");
5749         } else {
5750             print "extracting archive '$archive'\n";
5751             run_command($cmd);
5752         }
5753
5754         return if $opts->{info};
5755
5756         # read new mapping
5757         my $map = {};
5758         my $statfile = "$tmpdir/qmrestore.stat";
5759         if (my $fd = IO::File->new($statfile, "r")) {
5760             while (defined (my $line = <$fd>)) {
5761                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5762                     $map->{$1} = $2 if $1;
5763                 } else {
5764                     print STDERR "unable to parse line in statfile - $line\n";
5765                 }
5766             }
5767             $fd->close();
5768         }
5769
5770         my $confsrc = "$tmpdir/qemu-server.conf";
5771
5772         my $srcfd = new IO::File($confsrc, "r") ||
5773             die "unable to open file '$confsrc'\n";
5774
5775         my $outfd = new IO::File ($tmpfn, "w") ||
5776             die "unable to write config for VM $vmid\n";
5777
5778         my $cookie = { netcount => 0 };
5779         while (defined (my $line = <$srcfd>)) {
5780             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5781         }
5782
5783         $srcfd->close();
5784         $outfd->close();
5785     };
5786     my $err = $@;
5787
5788     if ($err) {
5789
5790         unlink $tmpfn;
5791
5792         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5793
5794         die $err;
5795     }
5796
5797     rmtree $tmpdir;
5798
5799     rename $tmpfn, $conffile ||
5800         die "unable to commit configuration file '$conffile'\n";
5801
5802     PVE::Cluster::cfs_update(); # make sure we read new file
5803
5804     eval { rescan($vmid, 1); };
5805     warn $@ if $@;
5806 };
5807
5808 sub foreach_writable_storage {
5809     my ($conf, $func) = @_;
5810
5811     my $sidhash = {};
5812
5813     foreach my $ds (keys %$conf) {
5814         next if !is_valid_drivename($ds);
5815
5816         my $drive = parse_drive($ds, $conf->{$ds});
5817         next if !$drive;
5818         next if drive_is_cdrom($drive);
5819
5820         my $volid = $drive->{file};
5821
5822         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5823         $sidhash->{$sid} = $sid if $sid;
5824     }
5825
5826     foreach my $sid (sort keys %$sidhash) {
5827         &$func($sid);
5828     }
5829 }
5830
5831 sub do_snapshots_with_qemu {
5832     my ($storecfg, $volid) = @_;
5833
5834     my $storage_name = PVE::Storage::parse_volume_id($volid);
5835
5836     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5837         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5838         return 1;
5839     }
5840
5841     if ($volid =~ m/\.(qcow2|qed)$/){
5842         return 1;
5843     }
5844
5845     return undef;
5846 }
5847
5848 sub qga_check_running {
5849     my ($vmid) = @_;
5850
5851     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5852     if ($@) {
5853         warn "Qemu Guest Agent are not running - $@";
5854         return 0;
5855     }
5856     return 1;
5857 }
5858
5859 sub template_create {
5860     my ($vmid, $conf, $disk) = @_;
5861
5862     my $storecfg = PVE::Storage::config();
5863
5864     foreach_drive($conf, sub {
5865         my ($ds, $drive) = @_;
5866
5867         return if drive_is_cdrom($drive);
5868         return if $disk && $ds ne $disk;
5869
5870         my $volid = $drive->{file};
5871         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5872
5873         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5874         $drive->{file} = $voliddst;
5875         $conf->{$ds} = print_drive($vmid, $drive);
5876         PVE::QemuConfig->write_config($vmid, $conf);
5877     });
5878 }
5879
5880 sub qemu_img_convert {
5881     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5882
5883     my $storecfg = PVE::Storage::config();
5884     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5885     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5886
5887     if ($src_storeid && $dst_storeid) {
5888
5889         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5890
5891         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5892         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5893
5894         my $src_format = qemu_img_format($src_scfg, $src_volname);
5895         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5896
5897         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5898         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5899
5900         my $cmd = [];
5901         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5902         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5903         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5904         if ($is_zero_initialized) {
5905             push @$cmd, "zeroinit:$dst_path";
5906         } else {
5907             push @$cmd, $dst_path;
5908         }
5909
5910         my $parser = sub {
5911             my $line = shift;
5912             if($line =~ m/\((\S+)\/100\%\)/){
5913                 my $percent = $1;
5914                 my $transferred = int($size * $percent / 100);
5915                 my $remaining = $size - $transferred;
5916
5917                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5918             }
5919
5920         };
5921
5922         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5923         my $err = $@;
5924         die "copy failed: $err" if $err;
5925     }
5926 }
5927
5928 sub qemu_img_format {
5929     my ($scfg, $volname) = @_;
5930
5931     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5932         return $1;
5933     } else {
5934         return "raw";
5935     }
5936 }
5937
5938 sub qemu_drive_mirror {
5939     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5940
5941     my $storecfg = PVE::Storage::config();
5942     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5943
5944     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5945
5946     my $format = qemu_img_format($dst_scfg, $dst_volname);
5947
5948     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5949
5950     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5951
5952     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5953     $opts->{format} = $format if $format;
5954
5955     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5956
5957     my $finish_job = sub {
5958         while (1) {
5959             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5960             my $stat = @$stats[0];
5961             last if !$stat;
5962             sleep 1;
5963         }
5964     };
5965
5966     eval {
5967     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5968         while (1) {
5969             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5970             my $stat = @$stats[0];
5971             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5972             die "error job is not mirroring" if $stat->{type} ne "mirror";
5973
5974             my $busy = $stat->{busy};
5975             my $ready = $stat->{ready};
5976
5977             if (my $total = $stat->{len}) {
5978                 my $transferred = $stat->{offset} || 0;
5979                 my $remaining = $total - $transferred;
5980                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5981
5982                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5983             }
5984
5985
5986             if ($stat->{ready} eq 'true') {
5987
5988                 last if $vmiddst != $vmid;
5989
5990                 # try to switch the disk if source and destination are on the same guest
5991                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5992                 if (!$@) {
5993                     &$finish_job();
5994                     last;
5995                 }
5996                 die $@ if $@ !~ m/cannot be completed/;
5997             }
5998             sleep 1;
5999         }
6000
6001
6002     };
6003     my $err = $@;
6004
6005     my $cancel_job = sub {
6006         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6007         &$finish_job();
6008     };
6009
6010     if ($err) {
6011         eval { &$cancel_job(); };
6012         die "mirroring error: $err";
6013     }
6014
6015     if ($vmiddst != $vmid) {
6016         # if we clone a disk for a new target vm, we don't switch the disk
6017         &$cancel_job(); # so we call block-job-cancel
6018     }
6019 }
6020
6021 sub clone_disk {
6022     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6023         $newvmid, $storage, $format, $full, $newvollist) = @_;
6024
6025     my $newvolid;
6026
6027     if (!$full) {
6028         print "create linked clone of drive $drivename ($drive->{file})\n";
6029         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6030         push @$newvollist, $newvolid;
6031     } else {
6032         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6033         $storeid = $storage if $storage;
6034
6035         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6036         if (!$format) {
6037             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6038             $format = qemu_img_format($scfg, $volname);
6039         }
6040
6041         # test if requested format is supported - else use default
6042         my $supported = grep { $_ eq $format } @$validFormats;
6043         $format = $defFormat if !$supported;
6044
6045         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6046
6047         print "create full clone of drive $drivename ($drive->{file})\n";
6048         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6049         push @$newvollist, $newvolid;
6050
6051         PVE::Storage::activate_volumes($storecfg, $newvollist);
6052
6053         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6054         if (!$running || $snapname) {
6055             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6056         } else {
6057             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
6058         }
6059     }
6060
6061     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6062
6063     my $disk = $drive;
6064     $disk->{format} = undef;
6065     $disk->{file} = $newvolid;
6066     $disk->{size} = $size;
6067
6068     return $disk;
6069 }
6070
6071 # this only works if VM is running
6072 sub get_current_qemu_machine {
6073     my ($vmid) = @_;
6074
6075     my $cmd = { execute => 'query-machines', arguments => {} };
6076     my $res = vm_qmp_command($vmid, $cmd);
6077
6078     my ($current, $default);
6079     foreach my $e (@$res) {
6080         $default = $e->{name} if $e->{'is-default'};
6081         $current = $e->{name} if $e->{'is-current'};
6082     }
6083
6084     # fallback to the default machine if current is not supported by qemu
6085     return $current || $default || 'pc';
6086 }
6087
6088 sub qemu_machine_feature_enabled {
6089     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6090
6091     my $current_major;
6092     my $current_minor;
6093
6094     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6095
6096         $current_major = $3;
6097         $current_minor = $4;
6098
6099     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6100
6101         $current_major = $1;
6102         $current_minor = $2;
6103     }
6104
6105     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6106
6107
6108 }
6109
6110 sub qemu_machine_pxe {
6111     my ($vmid, $conf, $machine) = @_;
6112
6113     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6114
6115     foreach my $opt (keys %$conf) {
6116         next if $opt !~ m/^net(\d+)$/;
6117         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6118         next if !$net;
6119         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6120         return $machine.".pxe" if $romfile =~ m/pxe/;
6121         last;
6122     }
6123
6124     return $machine;
6125 }
6126
6127 sub qemu_use_old_bios_files {
6128     my ($machine_type) = @_;
6129
6130     return if !$machine_type;
6131
6132     my $use_old_bios_files = undef;
6133
6134     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6135         $machine_type = $1;
6136         $use_old_bios_files = 1;
6137     } else {
6138         my $kvmver = kvm_user_version();
6139         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6140         # load new efi bios files on migration. So this hack is required to allow
6141         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6142         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6143         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6144     }
6145
6146     return ($use_old_bios_files, $machine_type);
6147 }
6148
6149 sub lspci {
6150
6151     my $devices = {};
6152
6153     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6154             my (undef, $id, $function) = @_;
6155             my $res = { id => $id, function => $function};
6156             push @{$devices->{$id}}, $res;
6157     });
6158
6159     return $devices;
6160 }
6161
6162 sub vm_iothreads_list {
6163     my ($vmid) = @_;
6164
6165     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6166
6167     my $iothreads = {};
6168     foreach my $iothread (@$res) {
6169         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6170     }
6171
6172     return $iothreads;
6173 }
6174
6175 sub scsihw_infos {
6176     my ($conf, $drive) = @_;
6177
6178     my $maxdev = 0;
6179
6180     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6181         $maxdev = 7;
6182     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6183         $maxdev = 1;
6184     } else {
6185         $maxdev = 256;
6186     }
6187
6188     my $controller = int($drive->{index} / $maxdev);
6189     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6190
6191     return ($maxdev, $controller, $controller_prefix);
6192 }
6193
6194 # bash completion helper
6195
6196 sub complete_backup_archives {
6197     my ($cmdname, $pname, $cvalue) = @_;
6198
6199     my $cfg = PVE::Storage::config();
6200
6201     my $storeid;
6202
6203     if ($cvalue =~ m/^([^:]+):/) {
6204         $storeid = $1;
6205     }
6206
6207     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6208
6209     my $res = [];
6210     foreach my $id (keys %$data) {
6211         foreach my $item (@{$data->{$id}}) {
6212             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6213             push @$res, $item->{volid} if defined($item->{volid});
6214         }
6215     }
6216
6217     return $res;
6218 }
6219
6220 my $complete_vmid_full = sub {
6221     my ($running) = @_;
6222
6223     my $idlist = vmstatus();
6224
6225     my $res = [];
6226
6227     foreach my $id (keys %$idlist) {
6228         my $d = $idlist->{$id};
6229         if (defined($running)) {
6230             next if $d->{template};
6231             next if $running && $d->{status} ne 'running';
6232             next if !$running && $d->{status} eq 'running';
6233         }
6234         push @$res, $id;
6235
6236     }
6237     return $res;
6238 };
6239
6240 sub complete_vmid {
6241     return &$complete_vmid_full();
6242 }
6243
6244 sub complete_vmid_stopped {
6245     return &$complete_vmid_full(0);
6246 }
6247
6248 sub complete_vmid_running {
6249     return &$complete_vmid_full(1);
6250 }
6251
6252 sub complete_storage {
6253
6254     my $cfg = PVE::Storage::config();
6255     my $ids = $cfg->{ids};
6256
6257     my $res = [];
6258     foreach my $sid (keys %$ids) {
6259         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6260         next if !$ids->{$sid}->{content}->{images};
6261         push @$res, $sid;
6262     }
6263
6264     return $res;
6265 }
6266
6267 1;