PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35 use URI::Escape;
  36
  37 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  38
  39 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  40
  41 # Note about locking: we use flock on the config file protect
  42 # against concurent actions.
  43 # Aditionaly, we have a 'lock' setting in the config file. This
  44 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  45 # allowed when such lock is set. But you can ignore this kind of
  46 # lock with the --skiplock flag.
  47
  48 cfs_register_file('/qemu-server/',
  49                   \&parse_vm_config,
  50                   \&write_vm_config);
  51
  52 PVE::JSONSchema::register_standard_option('skiplock', {
  53     description => "Ignore locks - only root is allowed to use this option.",
  54     type => 'boolean',
  55     optional => 1,
  56 });
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 #no warnings 'redefine';
  72
  73 sub cgroups_write {
  74    my ($controller, $vmid, $option, $value) = @_;
  75
  76    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  77    PVE::ProcFSTools::write_proc_entry($path, $value);
  78
  79 }
  80
  81 my $nodename = PVE::INotify::nodename();
  82
  83 mkdir "/etc/pve/nodes/$nodename";
  84 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  85 mkdir $confdir;
  86
  87 my $var_run_tmpdir = "/var/run/qemu-server";
  88 mkdir $var_run_tmpdir;
  89
  90 my $lock_dir = "/var/lock/qemu-server";
  91 mkdir $lock_dir;
  92
  93 my $pcisysfs = "/sys/bus/pci";
  94
  95 my $cpu_vendor_list = {
  96     # Intel CPUs
  97     486 => 'GenuineIntel',
  98     pentium => 'GenuineIntel',
  99     pentium2  => 'GenuineIntel',
 100     pentium3  => 'GenuineIntel',
 101     coreduo => 'GenuineIntel',
 102     core2duo => 'GenuineIntel',
 103     Conroe  => 'GenuineIntel',
 104     Penryn  => 'GenuineIntel',
 105     Nehalem  => 'GenuineIntel',
 106     Westmere => 'GenuineIntel',
 107     SandyBridge => 'GenuineIntel',
 108     IvyBridge => 'GenuineIntel',
 109     Haswell => 'GenuineIntel',
 110     'Haswell-noTSX' => 'GenuineIntel',
 111     Broadwell => 'GenuineIntel',
 112     'Broadwell-noTSX' => 'GenuineIntel',
 113
 114     # AMD CPUs
 115     athlon => 'AuthenticAMD',
 116     phenom  => 'AuthenticAMD',
 117     Opteron_G1  => 'AuthenticAMD',
 118     Opteron_G2  => 'AuthenticAMD',
 119     Opteron_G3  => 'AuthenticAMD',
 120     Opteron_G4  => 'AuthenticAMD',
 121     Opteron_G5  => 'AuthenticAMD',
 122
 123     # generic types, use vendor from host node
 124     host => 'default',
 125     kvm32 => 'default',
 126     kvm64 => 'default',
 127     qemu32 => 'default',
 128     qemu64 => 'default',
 129 };
 130
 131 my $cpu_fmt = {
 132     cputype => {
 133         description => "Emulated CPU type.",
 134         type => 'string',
 135         enum => [ sort keys %$cpu_vendor_list ],
 136         format_description => 'cputype',
 137         default => 'kvm64',
 138         default_key => 1,
 139     },
 140     hidden => {
 141         description => "Do not identify as a KVM virtual machine.",
 142         type => 'boolean',
 143         optional => 1,
 144         default => 0
 145     },
 146 };
 147
 148 my $watchdog_fmt = {
 149     model => {
 150         default_key => 1,
 151         type => 'string',
 152         enum => [qw(i6300esb ib700)],
 153         description => "Watchdog type to emulate.",
 154         default => 'i6300esb',
 155         optional => 1,
 156     },
 157     action => {
 158         type => 'string',
 159         enum => [qw(reset shutdown poweroff pause debug none)],
 160         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 161         optional => 1,
 162     },
 163 };
 164 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 165
 166 my $confdesc = {
 167     onboot => {
 168         optional => 1,
 169         type => 'boolean',
 170         description => "Specifies whether a VM will be started during system bootup.",
 171         default => 0,
 172     },
 173     autostart => {
 174         optional => 1,
 175         type => 'boolean',
 176         description => "Automatic restart after crash (currently ignored).",
 177         default => 0,
 178     },
 179     hotplug => {
 180         optional => 1,
 181         type => 'string', format => 'pve-hotplug-features',
 182         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 183         default => 'network,disk,usb',
 184     },
 185     reboot => {
 186         optional => 1,
 187         type => 'boolean',
 188         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 189         default => 1,
 190     },
 191     lock => {
 192         optional => 1,
 193         type => 'string',
 194         description => "Lock/unlock the VM.",
 195         enum => [qw(migrate backup snapshot rollback)],
 196     },
 197     cpulimit => {
 198         optional => 1,
 199         type => 'number',
 200         description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 201         minimum => 0,
 202         maximum => 128,
 203         default => 0,
 204     },
 205     cpuunits => {
 206         optional => 1,
 207         type => 'integer',
 208         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 209         minimum => 0,
 210         maximum => 500000,
 211         default => 1000,
 212     },
 213     memory => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 217         minimum => 16,
 218         default => 512,
 219     },
 220     balloon => {
 221         optional => 1,
 222         type => 'integer',
 223         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 224         minimum => 0,
 225     },
 226     shares => {
 227         optional => 1,
 228         type => 'integer',
 229         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 230         minimum => 0,
 231         maximum => 50000,
 232         default => 1000,
 233     },
 234     keyboard => {
 235         optional => 1,
 236         type => 'string',
 237         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 238         enum => PVE::Tools::kvmkeymaplist(),
 239         default => 'en-us',
 240     },
 241     name => {
 242         optional => 1,
 243         type => 'string', format => 'dns-name',
 244         description => "Set a name for the VM. Only used on the configuration web interface.",
 245     },
 246     scsihw => {
 247         optional => 1,
 248         type => 'string',
 249         description => "scsi controller model",
 250         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 251         default => 'lsi',
 252     },
 253     description => {
 254         optional => 1,
 255         type => 'string',
 256         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 257     },
 258     ostype => {
 259         optional => 1,
 260         type => 'string',
 261         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 262         description => <<EODESC,
 263 Used to enable special optimization/features for specific
 264 operating systems:
 265
 266 other  => unspecified OS
 267 wxp    => Microsoft Windows XP
 268 w2k    => Microsoft Windows 2000
 269 w2k3   => Microsoft Windows 2003
 270 w2k8   => Microsoft Windows 2008
 271 wvista => Microsoft Windows Vista
 272 win7   => Microsoft Windows 7
 273 win8   => Microsoft Windows 8/2012
 274 l24    => Linux 2.4 Kernel
 275 l26    => Linux 2.6/3.X Kernel
 276 solaris => solaris/opensolaris/openindiania kernel
 277
 278 other|l24|l26|solaris                       ... no special behaviour
 279 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 280 EODESC
 281     },
 282     boot => {
 283         optional => 1,
 284         type => 'string',
 285         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 286         pattern => '[acdn]{1,4}',
 287         default => 'cdn',
 288     },
 289     bootdisk => {
 290         optional => 1,
 291         type => 'string', format => 'pve-qm-bootdisk',
 292         description => "Enable booting from specified disk.",
 293         pattern => '(ide|sata|scsi|virtio)\d+',
 294     },
 295     smp => {
 296         optional => 1,
 297         type => 'integer',
 298         description => "The number of CPUs. Please use option -sockets instead.",
 299         minimum => 1,
 300         default => 1,
 301     },
 302     sockets => {
 303         optional => 1,
 304         type => 'integer',
 305         description => "The number of CPU sockets.",
 306         minimum => 1,
 307         default => 1,
 308     },
 309     cores => {
 310         optional => 1,
 311         type => 'integer',
 312         description => "The number of cores per socket.",
 313         minimum => 1,
 314         default => 1,
 315     },
 316     numa => {
 317         optional => 1,
 318         type => 'boolean',
 319         description => "Enable/disable NUMA.",
 320         default => 0,
 321     },
 322     vcpus => {
 323         optional => 1,
 324         type => 'integer',
 325         description => "Number of hotplugged vcpus.",
 326         minimum => 1,
 327         default => 0,
 328     },
 329     acpi => {
 330         optional => 1,
 331         type => 'boolean',
 332         description => "Enable/disable ACPI.",
 333         default => 1,
 334     },
 335     agent => {
 336         optional => 1,
 337         type => 'boolean',
 338         description => "Enable/disable Qemu GuestAgent.",
 339         default => 0,
 340     },
 341     kvm => {
 342         optional => 1,
 343         type => 'boolean',
 344         description => "Enable/disable KVM hardware virtualization.",
 345         default => 1,
 346     },
 347     tdf => {
 348         optional => 1,
 349         type => 'boolean',
 350         description => "Enable/disable time drift fix.",
 351         default => 0,
 352     },
 353     localtime => {
 354         optional => 1,
 355         type => 'boolean',
 356         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 357     },
 358     freeze => {
 359         optional => 1,
 360         type => 'boolean',
 361         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 362     },
 363     vga => {
 364         optional => 1,
 365         type => 'string',
 366         description => "Select the VGA type. If you want to use high resolution" .
 367             " modes (>= 1280x1024x16) then you should use the options " .
 368             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 369             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 370             "display sever. For win* OS you can select how many independent " .
 371             "displays you want, Linux guests can add displays them self. " .
 372             "You can also run without any graphic card, using a serial device" .
 373             " as terminal.",
 374         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 375     },
 376     watchdog => {
 377         optional => 1,
 378         type => 'string', format => 'pve-qm-watchdog',
 379         description => "Create a virtual hardware watchdog device. Once enabled" .
 380             " (by a guest action), the watchdog must be periodically polled " .
 381             "by an agent inside the guest or else the watchdog will reset " .
 382             "the guest (or execute the respective action specified)",
 383     },
 384     startdate => {
 385         optional => 1,
 386         type => 'string',
 387         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 388         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 389         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 390         default => 'now',
 391     },
 392     startup =>  get_standard_option('pve-startup-order'),
 393     template => {
 394         optional => 1,
 395         type => 'boolean',
 396         description => "Enable/disable Template.",
 397         default => 0,
 398     },
 399     args => {
 400         optional => 1,
 401         type => 'string',
 402         description => <<EODESCR,
 403 Arbitrary arguments passed to kvm, for example:
 404
 405 args: -no-reboot -no-hpet
 406
 407 NOTE: this option is for experts only.
 408 EODESCR
 409     },
 410     tablet => {
 411         optional => 1,
 412         type => 'boolean',
 413         default => 1,
 414         description => "Enable/disable the USB tablet device. This device is " .
 415             "usually needed to allow absolute mouse positioning with VNC. " .
 416             "Else the mouse runs out of sync with normal VNC clients. " .
 417             "If you're running lots of console-only guests on one host, " .
 418             "you may consider disabling this to save some context switches. " .
 419             "This is turned off by default if you use spice (-vga=qxl).",
 420     },
 421     migrate_speed => {
 422         optional => 1,
 423         type => 'integer',
 424         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 425         minimum => 0,
 426         default => 0,
 427     },
 428     migrate_downtime => {
 429         optional => 1,
 430         type => 'number',
 431         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 432         minimum => 0,
 433         default => 0.1,
 434     },
 435     cdrom => {
 436         optional => 1,
 437         type => 'string', format => 'pve-qm-drive',
 438         typetext => 'volume',
 439         description => "This is an alias for option -ide2",
 440     },
 441     cpu => {
 442         optional => 1,
 443         description => "Emulated CPU type.",
 444         type => 'string',
 445         format => $cpu_fmt,
 446     },
 447     parent => get_standard_option('pve-snapshot-name', {
 448         optional => 1,
 449         description => "Parent snapshot name. This is used internally, and should not be modified.",
 450     }),
 451     snaptime => {
 452         optional => 1,
 453         description => "Timestamp for snapshots.",
 454         type => 'integer',
 455         minimum => 0,
 456     },
 457     vmstate => {
 458         optional => 1,
 459         type => 'string', format => 'pve-volume-id',
 460         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 461     },
 462     machine => {
 463         description => "Specific the Qemu machine type.",
 464         type => 'string',
 465         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 466         maxLength => 40,
 467         optional => 1,
 468     },
 469     smbios1 => {
 470         description => "Specify SMBIOS type 1 fields.",
 471         type => 'string', format => 'pve-qm-smbios1',
 472         maxLength => 256,
 473         optional => 1,
 474     },
 475     protection => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 479         default => 0,
 480     },
 481     bios => {
 482         optional => 1,
 483         type => 'string',
 484         enum => [ qw(seabios ovmf) ],
 485         description => "Select BIOS implementation.",
 486         default => 'seabios',
 487     },
 488 };
 489
 490 # what about other qemu settings ?
 491 #cpu => 'string',
 492 #machine => 'string',
 493 #fda => 'file',
 494 #fdb => 'file',
 495 #mtdblock => 'file',
 496 #sd => 'file',
 497 #pflash => 'file',
 498 #snapshot => 'bool',
 499 #bootp => 'file',
 500 ##tftp => 'dir',
 501 ##smb => 'dir',
 502 #kernel => 'file',
 503 #append => 'string',
 504 #initrd => 'file',
 505 ##soundhw => 'string',
 506
 507 while (my ($k, $v) = each %$confdesc) {
 508     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 509 }
 510
 511 my $MAX_IDE_DISKS = 4;
 512 my $MAX_SCSI_DISKS = 14;
 513 my $MAX_VIRTIO_DISKS = 16;
 514 my $MAX_SATA_DISKS = 6;
 515 my $MAX_USB_DEVICES = 5;
 516 my $MAX_NETS = 32;
 517 my $MAX_UNUSED_DISKS = 8;
 518 my $MAX_HOSTPCI_DEVICES = 4;
 519 my $MAX_SERIAL_PORTS = 4;
 520 my $MAX_PARALLEL_PORTS = 3;
 521 my $MAX_NUMA = 8;
 522 my $MAX_MEM = 4194304;
 523 my $STATICMEM = 1024;
 524
 525 my $numa_fmt = {
 526     cpus => {
 527         type => "string",
 528         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 529         description => "CPUs accessing this numa node.",
 530         format_description => "id[-id];...",
 531     },
 532     memory => {
 533         type => "number",
 534         description => "Amount of memory this numa node provides.",
 535         format_description => "mb",
 536         optional => 1,
 537     },
 538     hostnodes => {
 539         type => "string",
 540         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 541         description => "host numa nodes to use",
 542         format_description => "id[-id];...",
 543         optional => 1,
 544     },
 545     policy => {
 546         type => 'string',
 547         enum => [qw(preferred bind interleave)],
 548         format_description => 'preferred|bind|interleave',
 549         description => "numa allocation policy.",
 550         optional => 1,
 551     },
 552 };
 553 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 554 my $numadesc = {
 555     optional => 1,
 556     type => 'string', format => $numa_fmt,
 557     description => "numa topology",
 558 };
 559 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 560
 561 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 562     $confdesc->{"numa$i"} = $numadesc;
 563 }
 564
 565 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 566                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 567                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 568 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 569
 570 my $net_fmt = {
 571     macaddr => {
 572         type => 'string',
 573         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 574         description => "MAC address",
 575         format_description => "XX:XX:XX:XX:XX:XX",
 576         optional => 1,
 577     },
 578     model => { alias => 'macaddr', default_key => 1 },
 579     (map { $_ => { group => 'model' } } @$nic_model_list),
 580     bridge => {
 581         type => 'string',
 582         description => 'Bridge to attach the network device to.',
 583         format_description => 'bridge',
 584         optional => 1,
 585     },
 586     queues => {
 587         type => 'integer',
 588         minimum => 0, maximum => 16,
 589         description => 'Number of packet queues to be used on the device.',
 590         format_description => 'number',
 591         optional => 1,
 592     },
 593     rate => {
 594         type => 'number',
 595         minimum => 0,
 596         description => 'Rate limit in mbps as floating point number.',
 597         format_description => 'mbps',
 598         optional => 1,
 599     },
 600     tag => {
 601         type => 'integer',
 602         minimum => 2, maximum => 4094,
 603         description => 'VLAN tag to apply to packets on this interface.',
 604         format_description => 'vlanid',
 605         optional => 1,
 606     },
 607     trunks => {
 608         type => 'string',
 609         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 610         description => 'VLAN trunks to pass through this interface.',
 611         format_description => 'id;id...',
 612         optional => 1,
 613     },
 614     firewall => {
 615         type => 'boolean',
 616         description => 'Whether this interface should be protected by the firewall.',
 617         format_description => '0|1',
 618         optional => 1,
 619     },
 620     link_down => {
 621         type => 'boolean',
 622         description => 'Whether this interface should be DISconnected (like pulling the plug).',
 623         format_description => '0|1',
 624         optional => 1,
 625     },
 626 };
 627 my $netdesc = {
 628     optional => 1,
 629     type => 'string', format => 'pve-qm-net',
 630     description => <<EODESCR,
 631 Specify network devices.
 632
 633 MODEL is one of: $nic_model_list_txt
 634
 635 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 636 automatically generated if not specified.
 637
 638 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 639
 640 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 641
 642 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 643
 644 10.0.2.2   Gateway
 645 10.0.2.3   DNS Server
 646 10.0.2.4   SMB Server
 647
 648 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 649
 650 EODESCR
 651 };
 652 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 653
 654 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 655     $confdesc->{"net$i"} = $netdesc;
 656 }
 657
 658 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 659 sub verify_volume_id_or_qm_path {
 660     my ($volid, $noerr) = @_;
 661
 662     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 663         return $volid;
 664     }
 665
 666     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 667     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 668     if ($@) {
 669         return undef if $noerr;
 670         die $@;
 671     }
 672     return $volid;
 673 }
 674
 675 my $drivename_hash;
 676
 677 my %drivedesc_base = (
 678     volume => { alias => 'file' },
 679     file => {
 680         type => 'string',
 681         format => 'pve-volume-id-or-qm-path',
 682         default_key => 1,
 683         format_description => 'volume',
 684         description => "The drive's backing volume.",
 685     },
 686     media => {
 687         type => 'string',
 688         format_description => 'cdrom|disk',
 689         enum => [qw(cdrom disk)],
 690         description => "The drive's media type.",
 691         default => 'disk',
 692         optional => 1
 693     },
 694     cyls => {
 695         type => 'integer',
 696         format_description => 'count',
 697         description => "Force the drive's physical geometry to have a specific cylinder count.",
 698         optional => 1
 699     },
 700     heads => {
 701         type => 'integer',
 702         format_description => 'count',
 703         description => "Force the drive's physical geometry to have a specific head count.",
 704         optional => 1
 705     },
 706     secs => {
 707         type => 'integer',
 708         format_description => 'count',
 709         description => "Force the drive's physical geometry to have a specific sector count.",
 710         optional => 1
 711     },
 712     trans => {
 713         type => 'string',
 714         format_description => 'none|lba|auto',
 715         enum => [qw(none lba auto)],
 716         description => "Force disk geometry bios translation mode.",
 717         optional => 1,
 718     },
 719     snapshot => {
 720         type => 'boolean',
 721         format_description => 'on|off',
 722         description => "Whether the drive should be included when making snapshots.",
 723         optional => 1,
 724     },
 725     cache => {
 726         type => 'string',
 727         format_description => 'none|writethrough|writeback|unsafe|directsync',
 728         enum => [qw(none writethrough writeback unsafe directsync)],
 729         description => "The drive's cache mode",
 730         optional => 1,
 731     },
 732     format => {
 733         type => 'string',
 734         format_description => 'drive format',
 735         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 736         description => "The drive's backing file's data format.",
 737         optional => 1,
 738     },
 739     size => {
 740         type => 'string',
 741         format => 'disk-size',
 742         description => "Disk size. This is purely informational and has no effect.",
 743         optional => 1,
 744     },
 745     backup => {
 746         type => 'boolean',
 747         format_description => 'on|off',
 748         description => "Whether the drive should be included when making backups.",
 749         optional => 1,
 750     },
 751     werror => {
 752         type => 'string',
 753         format_description => 'enospc|ignore|report|stop',
 754         enum => [qw(enospc ignore report stop)],
 755         description => 'Write error action.',
 756         optional => 1,
 757     },
 758     aio => {
 759         type => 'string',
 760         format_description => 'native|threads',
 761         enum => [qw(native threads)],
 762         description => 'AIO type to use.',
 763         optional => 1,
 764     },
 765     discard => {
 766         type => 'string',
 767         format_description => 'ignore|on',
 768         enum => [qw(ignore on)],
 769         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 770         optional => 1,
 771     },
 772     detect_zeroes => {
 773         type => 'boolean',
 774         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 775         optional => 1,
 776     },
 777     serial => {
 778         type => 'string',
 779         format => 'urlencoded',
 780         format_description => 'serial',
 781         maxLength => 20*3, # *3 since it's %xx url enoded
 782         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 783         optional => 1,
 784     }
 785 );
 786
 787 my %rerror_fmt = (
 788     rerror => {
 789         type => 'string',
 790         format_description => 'ignore|report|stop',
 791         enum => [qw(ignore report stop)],
 792         description => 'Read error action.',
 793         optional => 1,
 794     },
 795 );
 796
 797 my %iothread_fmt = ( iothread => {
 798         type => 'boolean',
 799         format_description => 'off|on',
 800         description => "Whether to use iothreads for this drive",
 801         optional => 1,
 802 });
 803
 804 my %model_fmt = (
 805     model => {
 806         type => 'string',
 807         format => 'urlencoded',
 808         format_description => 'model',
 809         maxLength => 40*3, # *3 since it's %xx url enoded
 810         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 811         optional => 1,
 812     },
 813 );
 814
 815 my %queues_fmt = (
 816     queues => {
 817         type => 'integer',
 818         format_description => 'nbqueues',
 819         description => "Number of queues.",
 820         minimum => 2,
 821         optional => 1
 822     }
 823 );
 824
 825 my $add_throttle_desc = sub {
 826     my ($key, $type, $what, $size, $longsize) = @_;
 827     $drivedesc_base{$key} = {
 828         type => $type,
 829         format_description => $size,
 830         description => "Maximum $what speed in $longsize per second.",
 831         optional => 1,
 832     };
 833 };
 834 # throughput: (leaky bucket)
 835 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 836 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 837 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 838 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 839 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 840 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 841 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 842 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 843 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 844
 845 # pools: (pool of IO before throttling starts taking effect)
 846 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 847 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 848 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 849 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 850 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 851 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 852
 853 my $ide_fmt = {
 854     %drivedesc_base,
 855     %rerror_fmt,
 856     %model_fmt,
 857 };
 858
 859 my $idedesc = {
 860     optional => 1,
 861     type => 'string', format => $ide_fmt,
 862     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 863 };
 864 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 865
 866 my $scsi_fmt = {
 867     %drivedesc_base,
 868     %iothread_fmt,
 869     %queues_fmt,
 870 };
 871 my $scsidesc = {
 872     optional => 1,
 873     type => 'string', format => $scsi_fmt,
 874     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 875 };
 876 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 877
 878 my $sata_fmt = {
 879     %drivedesc_base,
 880     %rerror_fmt,
 881 };
 882 my $satadesc = {
 883     optional => 1,
 884     type => 'string', format => $sata_fmt,
 885     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 886 };
 887 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 888
 889 my $virtio_fmt = {
 890     %drivedesc_base,
 891     %iothread_fmt,
 892     %rerror_fmt,
 893 };
 894 my $virtiodesc = {
 895     optional => 1,
 896     type => 'string', format => $virtio_fmt,
 897     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 898 };
 899 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 900
 901 my $alldrive_fmt = {
 902     %drivedesc_base,
 903     %rerror_fmt,
 904     %iothread_fmt,
 905     %model_fmt,
 906     %queues_fmt,
 907 };
 908
 909 my $usb_fmt = {
 910     host => {
 911         default_key => 1,
 912         type => 'string', format => 'pve-qm-usb-device',
 913         format_description => 'HOSTUSBDEVICE|spice',
 914         description => 'The Host USB device or port or the value spice',
 915     },
 916     usb3 => {
 917         optional => 1,
 918         type => 'boolean',
 919         format_description => 'yes|no',
 920         description => 'Specifies whether if given host option is a USB3 device or port',
 921     },
 922 };
 923
 924 my $usbdesc = {
 925     optional => 1,
 926     type => 'string', format => $usb_fmt,
 927     description => <<EODESCR,
 928 Configure an USB device (n is 0 to 4). This can be used to
 929 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 930
 931 'bus-port(.port)*' (decimal numbers) or
 932 'vendor_id:product_id' (hexadeciaml numbers) or
 933 'spice'
 934
 935 You can use the 'lsusb -t' command to list existing usb devices.
 936
 937 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 938
 939 The value 'spice' can be used to add a usb redirection devices for spice.
 940
 941 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 942
 943 EODESCR
 944 };
 945 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 946
 947 # NOTE: the match-groups of this regex are used in parse_hostpci
 948 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 949 my $hostpci_fmt = {
 950     host => {
 951         default_key => 1,
 952         type => 'string',
 953         pattern => qr/$PCIRE(;$PCIRE)*/,
 954         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 955         description => "The PCI ID of a host's PCI device or a list of PCI virtual functions of the host.",
 956     },
 957     rombar => {
 958         type => 'boolean',
 959         optional => 1,
 960         default => 1,
 961     },
 962     pcie => {
 963         type => 'boolean',
 964         optional => 1,
 965         default => 0,
 966     },
 967     'x-vga' => {
 968         type => 'boolean',
 969         optional => 1,
 970         default => 0,
 971     },
 972 };
 973 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
 974
 975 my $hostpcidesc = {
 976         optional => 1,
 977         type => 'string', format => 'pve-qm-hostpci',
 978         description => <<EODESCR,
 979 Map host pci devices. HOSTPCIDEVICE syntax is:
 980
 981 'bus:dev.func' (hexadecimal numbers)
 982
 983 You can us the 'lspci' command to list existing pci devices.
 984
 985 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 986
 987 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 988
 989 Experimental: user reported problems with this option.
 990 EODESCR
 991 };
 992 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 993
 994 my $serialdesc = {
 995         optional => 1,
 996         type => 'string',
 997         pattern => '(/dev/.+|socket)',
 998         description =>  <<EODESCR,
 999 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
1000
1001 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1002
1003 Experimental: user reported problems with this option.
1004 EODESCR
1005 };
1006
1007 my $paralleldesc= {
1008         optional => 1,
1009         type => 'string',
1010         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1011         description =>  <<EODESCR,
1012 Map host parallel devices (n is 0 to 2).
1013
1014 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1015
1016 Experimental: user reported problems with this option.
1017 EODESCR
1018 };
1019
1020 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1021     $confdesc->{"parallel$i"} = $paralleldesc;
1022 }
1023
1024 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1025     $confdesc->{"serial$i"} = $serialdesc;
1026 }
1027
1028 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1029     $confdesc->{"hostpci$i"} = $hostpcidesc;
1030 }
1031
1032 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1033     $drivename_hash->{"ide$i"} = 1;
1034     $confdesc->{"ide$i"} = $idedesc;
1035 }
1036
1037 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1038     $drivename_hash->{"sata$i"} = 1;
1039     $confdesc->{"sata$i"} = $satadesc;
1040 }
1041
1042 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1043     $drivename_hash->{"scsi$i"} = 1;
1044     $confdesc->{"scsi$i"} = $scsidesc ;
1045 }
1046
1047 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1048     $drivename_hash->{"virtio$i"} = 1;
1049     $confdesc->{"virtio$i"} = $virtiodesc;
1050 }
1051
1052 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1053     $confdesc->{"usb$i"} = $usbdesc;
1054 }
1055
1056 my $unuseddesc = {
1057     optional => 1,
1058     type => 'string', format => 'pve-volume-id',
1059     description => "Reference to unused volumes.",
1060 };
1061
1062 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1063     $confdesc->{"unused$i"} = $unuseddesc;
1064 }
1065
1066 my $kvm_api_version = 0;
1067
1068 sub kvm_version {
1069
1070     return $kvm_api_version if $kvm_api_version;
1071
1072     my $fh = IO::File->new("</dev/kvm") ||
1073         return 0;
1074
1075     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1076         $kvm_api_version = $v;
1077     }
1078
1079     $fh->close();
1080
1081     return  $kvm_api_version;
1082 }
1083
1084 my $kvm_user_version;
1085
1086 sub kvm_user_version {
1087
1088     return $kvm_user_version if $kvm_user_version;
1089
1090     $kvm_user_version = 'unknown';
1091
1092     my $code = sub {
1093         my $line = shift;
1094         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1095             $kvm_user_version = $2;
1096         }
1097     };
1098
1099     eval { run_command("kvm -version", outfunc => $code); };
1100     warn $@ if $@;
1101
1102     return $kvm_user_version;
1103
1104 }
1105
1106 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1107
1108 sub valid_drive_names {
1109     # order is important - used to autoselect boot disk
1110     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1111             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1112             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1113             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1114 }
1115
1116 sub is_valid_drivename {
1117     my $dev = shift;
1118
1119     return defined($drivename_hash->{$dev});
1120 }
1121
1122 sub option_exists {
1123     my $key = shift;
1124     return defined($confdesc->{$key});
1125 }
1126
1127 sub nic_models {
1128     return $nic_model_list;
1129 }
1130
1131 sub os_list_description {
1132
1133     return {
1134         other => 'Other',
1135         wxp => 'Windows XP',
1136         w2k => 'Windows 2000',
1137         w2k3 =>, 'Windows 2003',
1138         w2k8 => 'Windows 2008',
1139         wvista => 'Windows Vista',
1140         win7 => 'Windows 7',
1141         win8 => 'Windows 8/2012',
1142         l24 => 'Linux 2.4',
1143         l26 => 'Linux 2.6',
1144     };
1145 }
1146
1147 my $cdrom_path;
1148
1149 sub get_cdrom_path {
1150
1151     return  $cdrom_path if $cdrom_path;
1152
1153     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1154     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1155     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1156 }
1157
1158 sub get_iso_path {
1159     my ($storecfg, $vmid, $cdrom) = @_;
1160
1161     if ($cdrom eq 'cdrom') {
1162         return get_cdrom_path();
1163     } elsif ($cdrom eq 'none') {
1164         return '';
1165     } elsif ($cdrom =~ m|^/|) {
1166         return $cdrom;
1167     } else {
1168         return PVE::Storage::path($storecfg, $cdrom);
1169     }
1170 }
1171
1172 # try to convert old style file names to volume IDs
1173 sub filename_to_volume_id {
1174     my ($vmid, $file, $media) = @_;
1175
1176     if (!($file eq 'none' || $file eq 'cdrom' ||
1177           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1178
1179         return undef if $file =~ m|/|;
1180
1181         if ($media && $media eq 'cdrom') {
1182             $file = "local:iso/$file";
1183         } else {
1184             $file = "local:$vmid/$file";
1185         }
1186     }
1187
1188     return $file;
1189 }
1190
1191 sub verify_media_type {
1192     my ($opt, $vtype, $media) = @_;
1193
1194     return if !$media;
1195
1196     my $etype;
1197     if ($media eq 'disk') {
1198         $etype = 'images';
1199     } elsif ($media eq 'cdrom') {
1200         $etype = 'iso';
1201     } else {
1202         die "internal error";
1203     }
1204
1205     return if ($vtype eq $etype);
1206
1207     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1208 }
1209
1210 sub cleanup_drive_path {
1211     my ($opt, $storecfg, $drive) = @_;
1212
1213     # try to convert filesystem paths to volume IDs
1214
1215     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1216         ($drive->{file} !~ m|^/dev/.+|) &&
1217         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1218         ($drive->{file} !~ m/^\d+$/)) {
1219         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1220         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1221         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1222         verify_media_type($opt, $vtype, $drive->{media});
1223         $drive->{file} = $volid;
1224     }
1225
1226     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1227 }
1228
1229 sub parse_hotplug_features {
1230     my ($data) = @_;
1231
1232     my $res = {};
1233
1234     return $res if $data eq '0';
1235
1236     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1237
1238     foreach my $feature (PVE::Tools::split_list($data)) {
1239         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1240             $res->{$1} = 1;
1241         } else {
1242             warn "ignoring unknown hotplug feature '$feature'\n";
1243         }
1244     }
1245     return $res;
1246 }
1247
1248 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1249 sub pve_verify_hotplug_features {
1250     my ($value, $noerr) = @_;
1251
1252     return $value if parse_hotplug_features($value);
1253
1254     return undef if $noerr;
1255
1256     die "unable to parse hotplug option\n";
1257 }
1258
1259 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1260 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1261 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1262 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1263 #        [,iothread=on][,serial=serial][,model=model]
1264
1265 sub parse_drive {
1266     my ($key, $data) = @_;
1267
1268     my ($interface, $index);
1269
1270     if ($key =~ m/^([^\d]+)(\d+)$/) {
1271         $interface = $1;
1272         $index = $2;
1273     } else {
1274         return undef;
1275     }
1276
1277     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1278                                      : $confdesc->{$key}->{format};
1279     if (!$desc) {
1280         warn "invalid drive key: $key\n";
1281         return undef;
1282     }
1283     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1284     return undef if !$res;
1285     $res->{interface} = $interface;
1286     $res->{index} = $index;
1287
1288     my $error = 0;
1289     foreach my $opt (qw(bps bps_rd bps_wr)) {
1290         if (my $bps = defined(delete $res->{$opt})) {
1291             if (defined($res->{"m$opt"})) {
1292                 warn "both $opt and m$opt specified\n";
1293                 ++$error;
1294                 next;
1295             }
1296             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1297         }
1298     }
1299     return undef if $error;
1300
1301     return undef if $res->{mbps_rd} && $res->{mbps};
1302     return undef if $res->{mbps_wr} && $res->{mbps};
1303     return undef if $res->{iops_rd} && $res->{iops};
1304     return undef if $res->{iops_wr} && $res->{iops};
1305
1306     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1307         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1308         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1309         return undef if $res->{interface} eq 'virtio';
1310     }
1311
1312     if (my $size = $res->{size}) {
1313         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1314     }
1315
1316     return $res;
1317 }
1318
1319 sub print_drive {
1320     my ($vmid, $drive) = @_;
1321     my $data = { %$drive };
1322     delete $data->{$_} for qw(index interface);
1323     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1324 }
1325
1326 sub scsi_inquiry {
1327     my($fh, $noerr) = @_;
1328
1329     my $SG_IO = 0x2285;
1330     my $SG_GET_VERSION_NUM = 0x2282;
1331
1332     my $versionbuf = "\x00" x 8;
1333     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1334     if (!$ret) {
1335         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1336         return undef;
1337     }
1338     my $version = unpack("I", $versionbuf);
1339     if ($version < 30000) {
1340         die "scsi generic interface too old\n"  if !$noerr;
1341         return undef;
1342     }
1343
1344     my $buf = "\x00" x 36;
1345     my $sensebuf = "\x00" x 8;
1346     my $cmd = pack("C x3 C x1", 0x12, 36);
1347
1348     # see /usr/include/scsi/sg.h
1349     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1350
1351     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1352                       length($sensebuf), 0, length($buf), $buf,
1353                       $cmd, $sensebuf, 6000);
1354
1355     $ret = ioctl($fh, $SG_IO, $packet);
1356     if (!$ret) {
1357         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1358         return undef;
1359     }
1360
1361     my @res = unpack($sg_io_hdr_t, $packet);
1362     if ($res[17] || $res[18]) {
1363         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1364         return undef;
1365     }
1366
1367     my $res = {};
1368     (my $byte0, my $byte1, $res->{vendor},
1369      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1370
1371     $res->{removable} = $byte1 & 128 ? 1 : 0;
1372     $res->{type} = $byte0 & 31;
1373
1374     return $res;
1375 }
1376
1377 sub path_is_scsi {
1378     my ($path) = @_;
1379
1380     my $fh = IO::File->new("+<$path") || return undef;
1381     my $res = scsi_inquiry($fh, 1);
1382     close($fh);
1383
1384     return $res;
1385 }
1386
1387 sub machine_type_is_q35 {
1388     my ($conf) = @_;
1389
1390     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1391 }
1392
1393 sub print_tabletdevice_full {
1394     my ($conf) = @_;
1395
1396     my $q35 = machine_type_is_q35($conf);
1397
1398     # we use uhci for old VMs because tablet driver was buggy in older qemu
1399     my $usbbus = $q35 ? "ehci" : "uhci";
1400
1401     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1402 }
1403
1404 sub print_drivedevice_full {
1405     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1406
1407     my $device = '';
1408     my $maxdev = 0;
1409
1410     if ($drive->{interface} eq 'virtio') {
1411         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1412         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1413         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1414     } elsif ($drive->{interface} eq 'scsi') {
1415
1416         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1417         my $unit = $drive->{index} % $maxdev;
1418         my $devicetype = 'hd';
1419         my $path = '';
1420         if (drive_is_cdrom($drive)) {
1421             $devicetype = 'cd';
1422         } else {
1423             if ($drive->{file} =~ m|^/|) {
1424                 $path = $drive->{file};
1425                 if (my $info = path_is_scsi($path)) {
1426                     if ($info->{type} == 0) {
1427                         $devicetype = 'block';
1428                     } elsif ($info->{type} == 1) { # tape
1429                         $devicetype = 'generic';
1430                     }
1431                 }
1432             } else {
1433                  $path = PVE::Storage::path($storecfg, $drive->{file});
1434             }
1435
1436             if($path =~ m/^iscsi\:\/\//){
1437                 $devicetype = 'generic';
1438             }
1439         }
1440
1441         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1442             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1443         } else {
1444             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1445         }
1446
1447     } elsif ($drive->{interface} eq 'ide'){
1448         $maxdev = 2;
1449         my $controller = int($drive->{index} / $maxdev);
1450         my $unit = $drive->{index} % $maxdev;
1451         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1452
1453         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1454         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1455             $model = URI::Escape::uri_unescape($model);
1456             $device .= ",model=$model";
1457         }
1458     } elsif ($drive->{interface} eq 'sata'){
1459         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1460         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1461         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1462     } elsif ($drive->{interface} eq 'usb') {
1463         die "implement me";
1464         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1465     } else {
1466         die "unsupported interface type";
1467     }
1468
1469     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1470
1471     return $device;
1472 }
1473
1474 sub get_initiator_name {
1475     my $initiator;
1476
1477     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1478     while (defined(my $line = <$fh>)) {
1479         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1480         $initiator = $1;
1481         last;
1482     }
1483     $fh->close();
1484
1485     return $initiator;
1486 }
1487
1488 sub print_drive_full {
1489     my ($storecfg, $vmid, $drive) = @_;
1490
1491     my $path;
1492     my $volid = $drive->{file};
1493     my $format;
1494
1495     if (drive_is_cdrom($drive)) {
1496         $path = get_iso_path($storecfg, $vmid, $volid);
1497     } else {
1498         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1499         if ($storeid) {
1500             $path = PVE::Storage::path($storecfg, $volid);
1501             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1502             $format = qemu_img_format($scfg, $volname);
1503         } else {
1504             $path = $volid;
1505             $format = "raw";
1506         }
1507    }
1508
1509     my $opts = '';
1510     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1511     foreach my $o (@qemu_drive_options) {
1512         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1513     }
1514     if (my $serial = $drive->{serial}) {
1515         $serial = URI::Escape::uri_unescape($serial);
1516         $opts .= ",serial=$serial";
1517     }
1518
1519     $opts .= ",format=$format" if $format && !$drive->{format};
1520
1521     foreach my $o (qw(bps bps_rd bps_wr)) {
1522         my $v = $drive->{"m$o"};
1523         $opts .= ",$o=" . int($v*1024*1024) if $v;
1524     }
1525
1526     my $cache_direct = 0;
1527
1528     if (my $cache = $drive->{cache}) {
1529         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1530     } elsif (!drive_is_cdrom($drive)) {
1531         $opts .= ",cache=none";
1532         $cache_direct = 1;
1533     }
1534
1535     # aio native works only with O_DIRECT
1536     if (!$drive->{aio}) {
1537         if($cache_direct) {
1538             $opts .= ",aio=native";
1539         } else {
1540             $opts .= ",aio=threads";
1541         }
1542     }
1543
1544     if (!drive_is_cdrom($drive)) {
1545         my $detectzeroes;
1546         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1547             $detectzeroes = 'off';
1548         } elsif ($drive->{discard}) {
1549             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1550         } else {
1551             # This used to be our default with discard not being specified:
1552             $detectzeroes = 'on';
1553         }
1554         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1555     }
1556
1557     my $pathinfo = $path ? "file=$path," : '';
1558
1559     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1560 }
1561
1562 sub print_netdevice_full {
1563     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1564
1565     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1566
1567     my $device = $net->{model};
1568     if ($net->{model} eq 'virtio') {
1569          $device = 'virtio-net-pci';
1570      };
1571
1572     my $pciaddr = print_pci_addr("$netid", $bridges);
1573     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1574     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1575         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1576         my $vectors = $net->{queues} * 2 + 2;
1577         $tmpstr .= ",vectors=$vectors,mq=on";
1578     }
1579     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1580
1581     if ($use_old_bios_files) {
1582         my $romfile;
1583         if ($device eq 'virtio-net-pci') {
1584             $romfile = 'pxe-virtio.rom';
1585         } elsif ($device eq 'e1000') {
1586             $romfile = 'pxe-e1000.rom';
1587         } elsif ($device eq 'ne2k') {
1588             $romfile = 'pxe-ne2k_pci.rom';
1589         } elsif ($device eq 'pcnet') {
1590             $romfile = 'pxe-pcnet.rom';
1591         } elsif ($device eq 'rtl8139') {
1592             $romfile = 'pxe-rtl8139.rom';
1593         }
1594         $tmpstr .= ",romfile=$romfile" if $romfile;
1595     }
1596
1597     return $tmpstr;
1598 }
1599
1600 sub print_netdev_full {
1601     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1602
1603     my $i = '';
1604     if ($netid =~ m/^net(\d+)$/) {
1605         $i = int($1);
1606     }
1607
1608     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1609
1610     my $ifname = "tap${vmid}i$i";
1611
1612     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1613     die "interface name '$ifname' is too long (max 15 character)\n"
1614         if length($ifname) >= 16;
1615
1616     my $vhostparam = '';
1617     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1618
1619     my $vmname = $conf->{name} || "vm$vmid";
1620
1621     my $netdev = "";
1622     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1623
1624     if ($net->{bridge}) {
1625         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1626     } else {
1627         $netdev = "type=user,id=$netid,hostname=$vmname";
1628     }
1629
1630     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1631
1632     return $netdev;
1633 }
1634
1635 sub drive_is_cdrom {
1636     my ($drive) = @_;
1637
1638     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1639
1640 }
1641
1642 sub parse_number_sets {
1643     my ($set) = @_;
1644     my $res = [];
1645     foreach my $part (split(/;/, $set)) {
1646         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1647             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1648             push @$res, [ $1, $2 ];
1649         } else {
1650             die "invalid range: $part\n";
1651         }
1652     }
1653     return $res;
1654 }
1655
1656 sub parse_numa {
1657     my ($data) = @_;
1658
1659     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1660     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1661     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1662     return $res;
1663 }
1664
1665 sub parse_hostpci {
1666     my ($value) = @_;
1667
1668     return undef if !$value;
1669
1670     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1671
1672     my @idlist = split(/;/, $res->{host});
1673     delete $res->{host};
1674     foreach my $id (@idlist) {
1675         if ($id =~ /^$PCIRE$/) {
1676             push @{$res->{pciid}}, { id => $1, function => ($2//'0') };
1677         } else {
1678             # should have been caught by parse_property_string already
1679             die "failed to parse PCI id: $id\n";
1680         }
1681     }
1682     return $res;
1683 }
1684
1685 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1686 sub parse_net {
1687     my ($data) = @_;
1688
1689     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1690     if ($@) {
1691         warn $@;
1692         return undef;
1693     }
1694     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1695     return $res;
1696 }
1697
1698 sub print_net {
1699     my $net = shift;
1700
1701     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1702 }
1703
1704 sub add_random_macs {
1705     my ($settings) = @_;
1706
1707     foreach my $opt (keys %$settings) {
1708         next if $opt !~ m/^net(\d+)$/;
1709         my $net = parse_net($settings->{$opt});
1710         next if !$net;
1711         $settings->{$opt} = print_net($net);
1712     }
1713 }
1714
1715 sub vm_is_volid_owner {
1716     my ($storecfg, $vmid, $volid) = @_;
1717
1718     if ($volid !~  m|^/|) {
1719         my ($path, $owner);
1720         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1721         if ($owner && ($owner == $vmid)) {
1722             return 1;
1723         }
1724     }
1725
1726     return undef;
1727 }
1728
1729 sub split_flagged_list {
1730     my $text = shift || '';
1731     $text =~ s/[,;]/ /g;
1732     $text =~ s/^\s+//;
1733     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1734 }
1735
1736 sub join_flagged_list {
1737     my ($how, $lst) = @_;
1738     join $how, map { $lst->{$_} . $_ } keys %$lst;
1739 }
1740
1741 sub vmconfig_delete_pending_option {
1742     my ($conf, $key, $force) = @_;
1743
1744     delete $conf->{pending}->{$key};
1745     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1746     $pending_delete_hash->{$key} = $force ? '!' : '';
1747     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1748 }
1749
1750 sub vmconfig_undelete_pending_option {
1751     my ($conf, $key) = @_;
1752
1753     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1754     delete $pending_delete_hash->{$key};
1755
1756     if (%$pending_delete_hash) {
1757         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1758     } else {
1759         delete $conf->{pending}->{delete};
1760     }
1761 }
1762
1763 sub vmconfig_register_unused_drive {
1764     my ($storecfg, $vmid, $conf, $drive) = @_;
1765
1766     if (!drive_is_cdrom($drive)) {
1767         my $volid = $drive->{file};
1768         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1769             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1770         }
1771     }
1772 }
1773
1774 sub vmconfig_cleanup_pending {
1775     my ($conf) = @_;
1776
1777     # remove pending changes when nothing changed
1778     my $changes;
1779     foreach my $opt (keys %{$conf->{pending}}) {
1780         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1781             $changes = 1;
1782             delete $conf->{pending}->{$opt};
1783         }
1784     }
1785
1786     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1787     my $pending_delete_hash = {};
1788     while (my ($opt, $force) = each %$current_delete_hash) {
1789         if (defined($conf->{$opt})) {
1790             $pending_delete_hash->{$opt} = $force;
1791         } else {
1792             $changes = 1;
1793         }
1794     }
1795
1796     if (%$pending_delete_hash) {
1797         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1798     } else {
1799         delete $conf->{pending}->{delete};
1800     }
1801
1802     return $changes;
1803 }
1804
1805 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1806 my $smbios1_fmt = {
1807     uuid => {
1808         type => 'string',
1809         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1810         format_description => 'UUID',
1811         optional => 1,
1812     },
1813     version => {
1814         type => 'string',
1815         pattern => '\S+',
1816         format_description => 'str',
1817         optional => 1,
1818     },
1819     serial => {
1820         type => 'string',
1821         pattern => '\S+',
1822         format_description => 'str',
1823         optional => 1,
1824     },
1825     manufacturer => {
1826         type => 'string',
1827         pattern => '\S+',
1828         format_description => 'name',
1829         optional => 1,
1830     },
1831     product => {
1832         type => 'string',
1833         pattern => '\S+',
1834         format_description => 'name',
1835         optional => 1,
1836     },
1837     sku => {
1838         type => 'string',
1839         pattern => '\S+',
1840         format_description => 'str',
1841         optional => 1,
1842     },
1843     family => {
1844         type => 'string',
1845         pattern => '\S+',
1846         format_description => 'str',
1847         optional => 1,
1848     },
1849 };
1850
1851 sub parse_smbios1 {
1852     my ($data) = @_;
1853
1854     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1855     warn $@ if $@;
1856     return $res;
1857 }
1858
1859 sub print_smbios1 {
1860     my ($smbios1) = @_;
1861     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1862 }
1863
1864 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1865
1866 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1867 sub verify_bootdisk {
1868     my ($value, $noerr) = @_;
1869
1870     return $value if is_valid_drivename($value);
1871
1872     return undef if $noerr;
1873
1874     die "invalid boot disk '$value'\n";
1875 }
1876
1877 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1878 sub verify_net {
1879     my ($value, $noerr) = @_;
1880
1881     return $value if parse_net($value);
1882
1883     return undef if $noerr;
1884
1885     die "unable to parse network options\n";
1886 }
1887
1888 sub parse_watchdog {
1889     my ($value) = @_;
1890
1891     return undef if !$value;
1892
1893     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1894     warn $@ if $@;
1895     return $res;
1896 }
1897
1898 sub parse_usb_device {
1899     my ($value) = @_;
1900
1901     return undef if !$value;
1902
1903     my $res = {};
1904     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1905         $res->{vendorid} = $2;
1906         $res->{productid} = $4;
1907     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1908         $res->{hostbus} = $1;
1909         $res->{hostport} = $2;
1910     } elsif ($value =~ m/^spice$/i) {
1911         $res->{spice} = 1;
1912     } else {
1913         return undef;
1914     }
1915
1916     return $res;
1917 }
1918
1919 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1920 sub verify_usb_device {
1921     my ($value, $noerr) = @_;
1922
1923     return $value if parse_usb_device($value);
1924
1925     return undef if $noerr;
1926
1927     die "unable to parse usb device\n";
1928 }
1929
1930 # add JSON properties for create and set function
1931 sub json_config_properties {
1932     my $prop = shift;
1933
1934     foreach my $opt (keys %$confdesc) {
1935         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1936         $prop->{$opt} = $confdesc->{$opt};
1937     }
1938
1939     return $prop;
1940 }
1941
1942 sub check_type {
1943     my ($key, $value) = @_;
1944
1945     die "unknown setting '$key'\n" if !$confdesc->{$key};
1946
1947     my $type = $confdesc->{$key}->{type};
1948
1949     if (!defined($value)) {
1950         die "got undefined value\n";
1951     }
1952
1953     if ($value =~ m/[\n\r]/) {
1954         die "property contains a line feed\n";
1955     }
1956
1957     if ($type eq 'boolean') {
1958         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1959         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1960         die "type check ('boolean') failed - got '$value'\n";
1961     } elsif ($type eq 'integer') {
1962         return int($1) if $value =~ m/^(\d+)$/;
1963         die "type check ('integer') failed - got '$value'\n";
1964     } elsif ($type eq 'number') {
1965         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1966         die "type check ('number') failed - got '$value'\n";
1967     } elsif ($type eq 'string') {
1968         if (my $fmt = $confdesc->{$key}->{format}) {
1969             if ($fmt eq 'pve-qm-drive') {
1970                 # special case - we need to pass $key to parse_drive()
1971                 my $drive = parse_drive($key, $value);
1972                 return $value if $drive;
1973                 die "unable to parse drive options\n";
1974             }
1975             PVE::JSONSchema::check_format($fmt, $value);
1976             return $value;
1977         }
1978         $value =~ s/^\"(.*)\"$/$1/;
1979         return $value;
1980     } else {
1981         die "internal error"
1982     }
1983 }
1984
1985 sub check_iommu_support{
1986     #fixme : need to check IOMMU support
1987     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1988
1989     my $iommu=1;
1990     return $iommu;
1991
1992 }
1993
1994 sub touch_config {
1995     my ($vmid) = @_;
1996
1997     my $conf = PVE::QemuConfig->config_file($vmid);
1998     utime undef, undef, $conf;
1999 }
2000
2001 sub destroy_vm {
2002     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2003
2004     my $conffile = PVE::QemuConfig->config_file($vmid);
2005
2006     my $conf = PVE::QemuConfig->load_config($vmid);
2007
2008     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2009
2010     # only remove disks owned by this VM
2011     foreach_drive($conf, sub {
2012         my ($ds, $drive) = @_;
2013
2014         return if drive_is_cdrom($drive);
2015
2016         my $volid = $drive->{file};
2017
2018         return if !$volid || $volid =~ m|^/|;
2019
2020         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2021         return if !$path || !$owner || ($owner != $vmid);
2022
2023         PVE::Storage::vdisk_free($storecfg, $volid);
2024     });
2025
2026     if ($keep_empty_config) {
2027         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2028     } else {
2029         unlink $conffile;
2030     }
2031
2032     # also remove unused disk
2033     eval {
2034         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2035
2036         eval {
2037             PVE::Storage::foreach_volid($dl, sub {
2038                 my ($volid, $sid, $volname, $d) = @_;
2039                 PVE::Storage::vdisk_free($storecfg, $volid);
2040             });
2041         };
2042         warn $@ if $@;
2043
2044     };
2045     warn $@ if $@;
2046 }
2047
2048 sub parse_vm_config {
2049     my ($filename, $raw) = @_;
2050
2051     return undef if !defined($raw);
2052
2053     my $res = {
2054         digest => Digest::SHA::sha1_hex($raw),
2055         snapshots => {},
2056         pending => {},
2057     };
2058
2059     $filename =~ m|/qemu-server/(\d+)\.conf$|
2060         || die "got strange filename '$filename'";
2061
2062     my $vmid = $1;
2063
2064     my $conf = $res;
2065     my $descr;
2066     my $section = '';
2067
2068     my @lines = split(/\n/, $raw);
2069     foreach my $line (@lines) {
2070         next if $line =~ m/^\s*$/;
2071
2072         if ($line =~ m/^\[PENDING\]\s*$/i) {
2073             $section = 'pending';
2074             if (defined($descr)) {
2075                 $descr =~ s/\s+$//;
2076                 $conf->{description} = $descr;
2077             }
2078             $descr = undef;
2079             $conf = $res->{$section} = {};
2080             next;
2081
2082         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2083             $section = $1;
2084             if (defined($descr)) {
2085                 $descr =~ s/\s+$//;
2086                 $conf->{description} = $descr;
2087             }
2088             $descr = undef;
2089             $conf = $res->{snapshots}->{$section} = {};
2090             next;
2091         }
2092
2093         if ($line =~ m/^\#(.*)\s*$/) {
2094             $descr = '' if !defined($descr);
2095             $descr .= PVE::Tools::decode_text($1) . "\n";
2096             next;
2097         }
2098
2099         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2100             $descr = '' if !defined($descr);
2101             $descr .= PVE::Tools::decode_text($2);
2102         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2103             $conf->{snapstate} = $1;
2104         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2105             my $key = $1;
2106             my $value = $2;
2107             $conf->{$key} = $value;
2108         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2109             my $value = $1;
2110             if ($section eq 'pending') {
2111                 $conf->{delete} = $value; # we parse this later
2112             } else {
2113                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2114             }
2115         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2116             my $key = $1;
2117             my $value = $2;
2118             eval { $value = check_type($key, $value); };
2119             if ($@) {
2120                 warn "vm $vmid - unable to parse value of '$key' - $@";
2121             } else {
2122                 my $fmt = $confdesc->{$key}->{format};
2123                 if ($fmt && $fmt eq 'pve-qm-drive') {
2124                     my $v = parse_drive($key, $value);
2125                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2126                         $v->{file} = $volid;
2127                         $value = print_drive($vmid, $v);
2128                     } else {
2129                         warn "vm $vmid - unable to parse value of '$key'\n";
2130                         next;
2131                     }
2132                 }
2133
2134                 if ($key eq 'cdrom') {
2135                     $conf->{ide2} = $value;
2136                 } else {
2137                     $conf->{$key} = $value;
2138                 }
2139             }
2140         }
2141     }
2142
2143     if (defined($descr)) {
2144         $descr =~ s/\s+$//;
2145         $conf->{description} = $descr;
2146     }
2147     delete $res->{snapstate}; # just to be sure
2148
2149     return $res;
2150 }
2151
2152 sub write_vm_config {
2153     my ($filename, $conf) = @_;
2154
2155     delete $conf->{snapstate}; # just to be sure
2156
2157     if ($conf->{cdrom}) {
2158         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2159         $conf->{ide2} = $conf->{cdrom};
2160         delete $conf->{cdrom};
2161     }
2162
2163     # we do not use 'smp' any longer
2164     if ($conf->{sockets}) {
2165         delete $conf->{smp};
2166     } elsif ($conf->{smp}) {
2167         $conf->{sockets} = $conf->{smp};
2168         delete $conf->{cores};
2169         delete $conf->{smp};
2170     }
2171
2172     my $used_volids = {};
2173
2174     my $cleanup_config = sub {
2175         my ($cref, $pending, $snapname) = @_;
2176
2177         foreach my $key (keys %$cref) {
2178             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2179                 $key eq 'snapstate' || $key eq 'pending';
2180             my $value = $cref->{$key};
2181             if ($key eq 'delete') {
2182                 die "propertry 'delete' is only allowed in [PENDING]\n"
2183                     if !$pending;
2184                 # fixme: check syntax?
2185                 next;
2186             }
2187             eval { $value = check_type($key, $value); };
2188             die "unable to parse value of '$key' - $@" if $@;
2189
2190             $cref->{$key} = $value;
2191
2192             if (!$snapname && is_valid_drivename($key)) {
2193                 my $drive = parse_drive($key, $value);
2194                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2195             }
2196         }
2197     };
2198
2199     &$cleanup_config($conf);
2200
2201     &$cleanup_config($conf->{pending}, 1);
2202
2203     foreach my $snapname (keys %{$conf->{snapshots}}) {
2204         die "internal error" if $snapname eq 'pending';
2205         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2206     }
2207
2208     # remove 'unusedX' settings if we re-add a volume
2209     foreach my $key (keys %$conf) {
2210         my $value = $conf->{$key};
2211         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2212             delete $conf->{$key};
2213         }
2214     }
2215
2216     my $generate_raw_config = sub {
2217         my ($conf, $pending) = @_;
2218
2219         my $raw = '';
2220
2221         # add description as comment to top of file
2222         if (defined(my $descr = $conf->{description})) {
2223             if ($descr) {
2224                 foreach my $cl (split(/\n/, $descr)) {
2225                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2226                 }
2227             } else {
2228                 $raw .= "#\n" if $pending;
2229             }
2230         }
2231
2232         foreach my $key (sort keys %$conf) {
2233             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2234             $raw .= "$key: $conf->{$key}\n";
2235         }
2236         return $raw;
2237     };
2238
2239     my $raw = &$generate_raw_config($conf);
2240
2241     if (scalar(keys %{$conf->{pending}})){
2242         $raw .= "\n[PENDING]\n";
2243         $raw .= &$generate_raw_config($conf->{pending}, 1);
2244     }
2245
2246     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2247         $raw .= "\n[$snapname]\n";
2248         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2249     }
2250
2251     return $raw;
2252 }
2253
2254 sub load_defaults {
2255
2256     my $res = {};
2257
2258     # we use static defaults from our JSON schema configuration
2259     foreach my $key (keys %$confdesc) {
2260         if (defined(my $default = $confdesc->{$key}->{default})) {
2261             $res->{$key} = $default;
2262         }
2263     }
2264
2265     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2266     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2267
2268     return $res;
2269 }
2270
2271 sub config_list {
2272     my $vmlist = PVE::Cluster::get_vmlist();
2273     my $res = {};
2274     return $res if !$vmlist || !$vmlist->{ids};
2275     my $ids = $vmlist->{ids};
2276
2277     foreach my $vmid (keys %$ids) {
2278         my $d = $ids->{$vmid};
2279         next if !$d->{node} || $d->{node} ne $nodename;
2280         next if !$d->{type} || $d->{type} ne 'qemu';
2281         $res->{$vmid}->{exists} = 1;
2282     }
2283     return $res;
2284 }
2285
2286 # test if VM uses local resources (to prevent migration)
2287 sub check_local_resources {
2288     my ($conf, $noerr) = @_;
2289
2290     my $loc_res = 0;
2291
2292     $loc_res = 1 if $conf->{hostusb}; # old syntax
2293     $loc_res = 1 if $conf->{hostpci}; # old syntax
2294
2295     foreach my $k (keys %$conf) {
2296         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2297         # sockets are safe: they will recreated be on the target side post-migrate
2298         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2299         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2300     }
2301
2302     die "VM uses local resources\n" if $loc_res && !$noerr;
2303
2304     return $loc_res;
2305 }
2306
2307 # check if used storages are available on all nodes (use by migrate)
2308 sub check_storage_availability {
2309     my ($storecfg, $conf, $node) = @_;
2310
2311     foreach_drive($conf, sub {
2312         my ($ds, $drive) = @_;
2313
2314         my $volid = $drive->{file};
2315         return if !$volid;
2316
2317         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2318         return if !$sid;
2319
2320         # check if storage is available on both nodes
2321         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2322         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2323    });
2324 }
2325
2326 # list nodes where all VM images are available (used by has_feature API)
2327 sub shared_nodes {
2328     my ($conf, $storecfg) = @_;
2329
2330     my $nodelist = PVE::Cluster::get_nodelist();
2331     my $nodehash = { map { $_ => 1 } @$nodelist };
2332     my $nodename = PVE::INotify::nodename();
2333
2334     foreach_drive($conf, sub {
2335         my ($ds, $drive) = @_;
2336
2337         my $volid = $drive->{file};
2338         return if !$volid;
2339
2340         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2341         if ($storeid) {
2342             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2343             if ($scfg->{disable}) {
2344                 $nodehash = {};
2345             } elsif (my $avail = $scfg->{nodes}) {
2346                 foreach my $node (keys %$nodehash) {
2347                     delete $nodehash->{$node} if !$avail->{$node};
2348                 }
2349             } elsif (!$scfg->{shared}) {
2350                 foreach my $node (keys %$nodehash) {
2351                     delete $nodehash->{$node} if $node ne $nodename
2352                 }
2353             }
2354         }
2355     });
2356
2357     return $nodehash
2358 }
2359
2360 sub check_cmdline {
2361     my ($pidfile, $pid) = @_;
2362
2363     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2364     if (defined($fh)) {
2365         my $line = <$fh>;
2366         $fh->close;
2367         return undef if !$line;
2368         my @param = split(/\0/, $line);
2369
2370         my $cmd = $param[0];
2371         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2372
2373         for (my $i = 0; $i < scalar (@param); $i++) {
2374             my $p = $param[$i];
2375             next if !$p;
2376             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2377                 my $p = $param[$i+1];
2378                 return 1 if $p && ($p eq $pidfile);
2379                 return undef;
2380             }
2381         }
2382     }
2383     return undef;
2384 }
2385
2386 sub check_running {
2387     my ($vmid, $nocheck, $node) = @_;
2388
2389     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2390
2391     die "unable to find configuration file for VM $vmid - no such machine\n"
2392         if !$nocheck && ! -f $filename;
2393
2394     my $pidfile = pidfile_name($vmid);
2395
2396     if (my $fd = IO::File->new("<$pidfile")) {
2397         my $st = stat($fd);
2398         my $line = <$fd>;
2399         close($fd);
2400
2401         my $mtime = $st->mtime;
2402         if ($mtime > time()) {
2403             warn "file '$filename' modified in future\n";
2404         }
2405
2406         if ($line =~ m/^(\d+)$/) {
2407             my $pid = $1;
2408             if (check_cmdline($pidfile, $pid)) {
2409                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2410                     return $pid;
2411                 }
2412             }
2413         }
2414     }
2415
2416     return undef;
2417 }
2418
2419 sub vzlist {
2420
2421     my $vzlist = config_list();
2422
2423     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2424
2425     while (defined(my $de = $fd->read)) {
2426         next if $de !~ m/^(\d+)\.pid$/;
2427         my $vmid = $1;
2428         next if !defined($vzlist->{$vmid});
2429         if (my $pid = check_running($vmid)) {
2430             $vzlist->{$vmid}->{pid} = $pid;
2431         }
2432     }
2433
2434     return $vzlist;
2435 }
2436
2437 sub disksize {
2438     my ($storecfg, $conf) = @_;
2439
2440     my $bootdisk = $conf->{bootdisk};
2441     return undef if !$bootdisk;
2442     return undef if !is_valid_drivename($bootdisk);
2443
2444     return undef if !$conf->{$bootdisk};
2445
2446     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2447     return undef if !defined($drive);
2448
2449     return undef if drive_is_cdrom($drive);
2450
2451     my $volid = $drive->{file};
2452     return undef if !$volid;
2453
2454     return $drive->{size};
2455 }
2456
2457 my $last_proc_pid_stat;
2458
2459 # get VM status information
2460 # This must be fast and should not block ($full == false)
2461 # We only query KVM using QMP if $full == true (this can be slow)
2462 sub vmstatus {
2463     my ($opt_vmid, $full) = @_;
2464
2465     my $res = {};
2466
2467     my $storecfg = PVE::Storage::config();
2468
2469     my $list = vzlist();
2470     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2471
2472     my $cpucount = $cpuinfo->{cpus} || 1;
2473
2474     foreach my $vmid (keys %$list) {
2475         next if $opt_vmid && ($vmid ne $opt_vmid);
2476
2477         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2478         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2479
2480         my $d = {};
2481         $d->{pid} = $list->{$vmid}->{pid};
2482
2483         # fixme: better status?
2484         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2485
2486         my $size = disksize($storecfg, $conf);
2487         if (defined($size)) {
2488             $d->{disk} = 0; # no info available
2489             $d->{maxdisk} = $size;
2490         } else {
2491             $d->{disk} = 0;
2492             $d->{maxdisk} = 0;
2493         }
2494
2495         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2496         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2497         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2498
2499         $d->{name} = $conf->{name} || "VM $vmid";
2500         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2501
2502         if ($conf->{balloon}) {
2503             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2504             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2505         }
2506
2507         $d->{uptime} = 0;
2508         $d->{cpu} = 0;
2509         $d->{mem} = 0;
2510
2511         $d->{netout} = 0;
2512         $d->{netin} = 0;
2513
2514         $d->{diskread} = 0;
2515         $d->{diskwrite} = 0;
2516
2517         $d->{template} = PVE::QemuConfig->is_template($conf);
2518
2519         $res->{$vmid} = $d;
2520     }
2521
2522     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2523     foreach my $dev (keys %$netdev) {
2524         next if $dev !~ m/^tap([1-9]\d*)i/;
2525         my $vmid = $1;
2526         my $d = $res->{$vmid};
2527         next if !$d;
2528
2529         $d->{netout} += $netdev->{$dev}->{receive};
2530         $d->{netin} += $netdev->{$dev}->{transmit};
2531
2532         if ($full) {
2533             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2534             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2535         }
2536
2537     }
2538
2539     my $ctime = gettimeofday;
2540
2541     foreach my $vmid (keys %$list) {
2542
2543         my $d = $res->{$vmid};
2544         my $pid = $d->{pid};
2545         next if !$pid;
2546
2547         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2548         next if !$pstat; # not running
2549
2550         my $used = $pstat->{utime} + $pstat->{stime};
2551
2552         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2553
2554         if ($pstat->{vsize}) {
2555             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2556         }
2557
2558         my $old = $last_proc_pid_stat->{$pid};
2559         if (!$old) {
2560             $last_proc_pid_stat->{$pid} = {
2561                 time => $ctime,
2562                 used => $used,
2563                 cpu => 0,
2564             };
2565             next;
2566         }
2567
2568         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2569
2570         if ($dtime > 1000) {
2571             my $dutime = $used -  $old->{used};
2572
2573             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2574             $last_proc_pid_stat->{$pid} = {
2575                 time => $ctime,
2576                 used => $used,
2577                 cpu => $d->{cpu},
2578             };
2579         } else {
2580             $d->{cpu} = $old->{cpu};
2581         }
2582     }
2583
2584     return $res if !$full;
2585
2586     my $qmpclient = PVE::QMPClient->new();
2587
2588     my $ballooncb = sub {
2589         my ($vmid, $resp) = @_;
2590
2591         my $info = $resp->{'return'};
2592         return if !$info->{max_mem};
2593
2594         my $d = $res->{$vmid};
2595
2596         # use memory assigned to VM
2597         $d->{maxmem} = $info->{max_mem};
2598         $d->{balloon} = $info->{actual};
2599
2600         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2601             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2602             $d->{freemem} = $info->{free_mem};
2603         }
2604
2605         $d->{ballooninfo} = $info;
2606     };
2607
2608     my $blockstatscb = sub {
2609         my ($vmid, $resp) = @_;
2610         my $data = $resp->{'return'} || [];
2611         my $totalrdbytes = 0;
2612         my $totalwrbytes = 0;
2613
2614         for my $blockstat (@$data) {
2615             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2616             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2617
2618             $blockstat->{device} =~ s/drive-//;
2619             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2620         }
2621         $res->{$vmid}->{diskread} = $totalrdbytes;
2622         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2623     };
2624
2625     my $statuscb = sub {
2626         my ($vmid, $resp) = @_;
2627
2628         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2629         # this fails if ballon driver is not loaded, so this must be
2630         # the last commnand (following command are aborted if this fails).
2631         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2632
2633         my $status = 'unknown';
2634         if (!defined($status = $resp->{'return'}->{status})) {
2635             warn "unable to get VM status\n";
2636             return;
2637         }
2638
2639         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2640     };
2641
2642     foreach my $vmid (keys %$list) {
2643         next if $opt_vmid && ($vmid ne $opt_vmid);
2644         next if !$res->{$vmid}->{pid}; # not running
2645         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2646     }
2647
2648     $qmpclient->queue_execute(undef, 1);
2649
2650     foreach my $vmid (keys %$list) {
2651         next if $opt_vmid && ($vmid ne $opt_vmid);
2652         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2653     }
2654
2655     return $res;
2656 }
2657
2658 sub foreach_dimm {
2659     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2660
2661     my $dimm_id = 0;
2662     my $current_size = 1024;
2663     my $dimm_size = 512;
2664     return if $current_size == $memory;
2665
2666     for (my $j = 0; $j < 8; $j++) {
2667         for (my $i = 0; $i < 32; $i++) {
2668             my $name = "dimm${dimm_id}";
2669             $dimm_id++;
2670             my $numanode = $i % $sockets;
2671             $current_size += $dimm_size;
2672             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2673             return  $current_size if $current_size >= $memory;
2674         }
2675         $dimm_size *= 2;
2676     }
2677 }
2678
2679 sub foreach_reverse_dimm {
2680     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2681
2682     my $dimm_id = 253;
2683     my $current_size = 4177920;
2684     my $dimm_size = 65536;
2685     return if $current_size == $memory;
2686
2687     for (my $j = 0; $j < 8; $j++) {
2688         for (my $i = 0; $i < 32; $i++) {
2689             my $name = "dimm${dimm_id}";
2690             $dimm_id--;
2691             my $numanode = $i % $sockets;
2692             $current_size -= $dimm_size;
2693             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2694             return  $current_size if $current_size <= $memory;
2695         }
2696         $dimm_size /= 2;
2697     }
2698 }
2699
2700 sub foreach_drive {
2701     my ($conf, $func) = @_;
2702
2703     foreach my $ds (valid_drive_names()) {
2704         next if !defined($conf->{$ds});
2705
2706         my $drive = parse_drive($ds, $conf->{$ds});
2707         next if !$drive;
2708
2709         &$func($ds, $drive);
2710     }
2711 }
2712
2713 sub foreach_volid {
2714     my ($conf, $func) = @_;
2715
2716     my $volhash = {};
2717
2718     my $test_volid = sub {
2719         my ($volid, $is_cdrom) = @_;
2720
2721         return if !$volid;
2722
2723         $volhash->{$volid} = $is_cdrom || 0;
2724     };
2725
2726     foreach_drive($conf, sub {
2727         my ($ds, $drive) = @_;
2728         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2729     });
2730
2731     foreach my $snapname (keys %{$conf->{snapshots}}) {
2732         my $snap = $conf->{snapshots}->{$snapname};
2733         &$test_volid($snap->{vmstate}, 0);
2734         foreach_drive($snap, sub {
2735             my ($ds, $drive) = @_;
2736             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2737         });
2738     }
2739
2740     foreach my $volid (keys %$volhash) {
2741         &$func($volid, $volhash->{$volid});
2742     }
2743 }
2744
2745 sub vga_conf_has_spice {
2746     my ($vga) = @_;
2747
2748     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2749
2750     return $1 || 1;
2751 }
2752
2753 sub config_to_command {
2754     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2755
2756     my $cmd = [];
2757     my $globalFlags = [];
2758     my $machineFlags = [];
2759     my $rtcFlags = [];
2760     my $cpuFlags = [];
2761     my $devices = [];
2762     my $pciaddr = '';
2763     my $bridges = {};
2764     my $kvmver = kvm_user_version();
2765     my $vernum = 0; # unknown
2766     my $ostype = $conf->{ostype};
2767     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2768         $vernum = $1*1000000+$2*1000;
2769     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2770         $vernum = $1*1000000+$2*1000+$3;
2771     }
2772
2773     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2774
2775     my $have_ovz = -f '/proc/vz/vestat';
2776
2777     my $q35 = machine_type_is_q35($conf);
2778     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2779     my $machine_type = $forcemachine || $conf->{machine};
2780     my $use_old_bios_files = undef;
2781     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2782
2783     my $cpuunits = defined($conf->{cpuunits}) ?
2784             $conf->{cpuunits} : $defaults->{cpuunits};
2785
2786     push @$cmd, '/usr/bin/systemd-run';
2787     push @$cmd, '--scope';
2788     push @$cmd, '--slice', "qemu";
2789     push @$cmd, '--unit', $vmid;
2790     # set KillMode=none, so that systemd don't kill those scopes
2791     # at shutdown (pve-manager service should stop the VMs instead)
2792     push @$cmd, '-p', "KillMode=none";
2793     push @$cmd, '-p', "CPUShares=$cpuunits";
2794     if ($conf->{cpulimit}) {
2795         my $cpulimit = int($conf->{cpulimit} * 100);
2796         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2797     }
2798
2799     push @$cmd, '/usr/bin/kvm';
2800
2801     push @$cmd, '-id', $vmid;
2802
2803     my $use_virtio = 0;
2804
2805     my $qmpsocket = qmp_socket($vmid);
2806     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2807     push @$cmd, '-mon', "chardev=qmp,mode=control";
2808
2809
2810     push @$cmd, '-pidfile' , pidfile_name($vmid);
2811
2812     push @$cmd, '-daemonize';
2813
2814     if ($conf->{smbios1}) {
2815         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2816     }
2817
2818     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2819         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2820         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2821         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2822         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2823         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2824         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2825     }
2826
2827     if ($q35) {
2828         # the q35 chipset support native usb2, so we enable usb controller
2829         # by default for this machine type
2830         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2831     } else {
2832         $pciaddr = print_pci_addr("piix3", $bridges);
2833         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2834
2835         my $use_usb2 = 0;
2836         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2837             next if !$conf->{"usb$i"};
2838             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2839             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2840             $use_usb2 = 1;
2841         }
2842         # include usb device config
2843         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2844     }
2845
2846     # add usb3 controller if needed
2847
2848     my $use_usb3 = 0;
2849     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2850         next if !$conf->{"usb$i"};
2851         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2852         next if !$d || !$d->{usb3};
2853         $use_usb3 = 1;
2854     }
2855
2856     $pciaddr = print_pci_addr("xhci", $bridges);
2857     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2858
2859     my $vga = $conf->{vga};
2860
2861     my $qxlnum = vga_conf_has_spice($vga);
2862     $vga = 'qxl' if $qxlnum;
2863
2864     if (!$vga) {
2865         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2866                                 $conf->{ostype} eq 'win7' ||
2867                                 $conf->{ostype} eq 'w2k8')) {
2868             $vga = 'std';
2869         } else {
2870             $vga = 'cirrus';
2871         }
2872     }
2873
2874     # enable absolute mouse coordinates (needed by vnc)
2875     my $tablet;
2876     if (defined($conf->{tablet})) {
2877         $tablet = $conf->{tablet};
2878     } else {
2879         $tablet = $defaults->{tablet};
2880         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2881         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2882     }
2883
2884     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2885
2886     my $kvm_off = 0;
2887     # host pci devices
2888     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2889         my $d = parse_hostpci($conf->{"hostpci$i"});
2890         next if !$d;
2891
2892         my $pcie = $d->{pcie};
2893         if($pcie){
2894             die "q35 machine model is not enabled" if !$q35;
2895             $pciaddr = print_pcie_addr("hostpci$i");
2896         }else{
2897             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2898         }
2899
2900         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2901         my $xvga = '';
2902         if ($d->{'x-vga'}) {
2903             $xvga = ',x-vga=on';
2904             $kvm_off = 1;
2905             $vga = 'none';
2906             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2907                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2908             }
2909             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2910                 $xvga = "";
2911             }
2912         }
2913         my $pcidevices = $d->{pciid};
2914         my $multifunction = 1 if @$pcidevices > 1;
2915
2916         my $j=0;
2917         foreach my $pcidevice (@$pcidevices) {
2918
2919             my $id = "hostpci$i";
2920             $id .= ".$j" if $multifunction;
2921             my $addr = $pciaddr;
2922             $addr .= ".$j" if $multifunction;
2923             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2924
2925             if($j == 0){
2926                 $devicestr .= "$rombar$xvga";
2927                 $devicestr .= ",multifunction=on" if $multifunction;
2928             }
2929
2930             push @$devices, '-device', $devicestr;
2931             $j++;
2932         }
2933     }
2934
2935     # usb devices
2936     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2937         next if !$conf->{"usb$i"};
2938         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2939         next if !$d;
2940
2941         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2942         my $usbbus = '';
2943         if (defined($d->{usb3}) && $d->{usb3}) {
2944             $usbbus = ',bus=xhci.0';
2945         }
2946
2947         if (defined($d->{host})) {
2948             $d = parse_usb_device($d->{host});
2949             if (defined($d->{vendorid}) && defined($d->{productid})) {
2950                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2951             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2952                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2953             } elsif (defined($d->{spice}) && $d->{spice}) {
2954                 # usb redir support for spice, currently no usb3
2955                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2956                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2957             }
2958         }
2959     }
2960
2961     # serial devices
2962     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2963         if (my $path = $conf->{"serial$i"}) {
2964             if ($path eq 'socket') {
2965                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2966                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2967                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2968             } else {
2969                 die "no such serial device\n" if ! -c $path;
2970                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2971                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2972             }
2973         }
2974     }
2975
2976     # parallel devices
2977     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2978         if (my $path = $conf->{"parallel$i"}) {
2979             die "no such parallel device\n" if ! -c $path;
2980             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2981             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2982             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2983         }
2984     }
2985
2986     my $vmname = $conf->{name} || "vm$vmid";
2987
2988     push @$cmd, '-name', $vmname;
2989
2990     my $sockets = 1;
2991     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2992     $sockets = $conf->{sockets} if  $conf->{sockets};
2993
2994     my $cores = $conf->{cores} || 1;
2995
2996     my $maxcpus = $sockets * $cores;
2997
2998     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2999
3000     my $allowed_vcpus = $cpuinfo->{cpus};
3001
3002     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3003         if ($allowed_vcpus < $maxcpus);
3004
3005     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3006
3007     push @$cmd, '-nodefaults';
3008
3009     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3010
3011     my $bootindex_hash = {};
3012     my $i = 1;
3013     foreach my $o (split(//, $bootorder)) {
3014         $bootindex_hash->{$o} = $i*100;
3015         $i++;
3016     }
3017
3018     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
3019
3020     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3021
3022     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3023
3024     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3025
3026     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3027         my $socket = vnc_socket($vmid);
3028         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3029     } else {
3030         push @$cmd, '-nographic';
3031     }
3032
3033     # time drift fix
3034     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3035
3036     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3037     my $useLocaltime = $conf->{localtime};
3038
3039     if ($ostype) {
3040         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3041
3042         if ($ostype =~ m/^w/) { # windows
3043             $useLocaltime = 1 if !defined($conf->{localtime});
3044
3045             # use time drift fix when acpi is enabled
3046             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3047                 $tdf = 1 if !defined($conf->{tdf});
3048             }
3049         }
3050
3051         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3052             $ostype eq 'wvista') {
3053             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3054             push @$cmd, '-no-hpet';
3055             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3056                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3057                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3058                 push @$cpuFlags , 'hv_time' if !$nokvm;
3059
3060             } else {
3061                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3062             }
3063         }
3064
3065         if ($ostype eq 'win7' || $ostype eq 'win8') {
3066             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3067         }
3068     }
3069
3070     push @$rtcFlags, 'driftfix=slew' if $tdf;
3071
3072     if ($nokvm) {
3073         push @$machineFlags, 'accel=tcg';
3074     } else {
3075         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3076     }
3077
3078     if ($machine_type) {
3079         push @$machineFlags, "type=${machine_type}";
3080     }
3081
3082     if ($conf->{startdate}) {
3083         push @$rtcFlags, "base=$conf->{startdate}";
3084     } elsif ($useLocaltime) {
3085         push @$rtcFlags, 'base=localtime';
3086     }
3087
3088     my $cpu = $nokvm ? "qemu64" : "kvm64";
3089     if (my $cputype = $conf->{cpu}) {
3090         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3091             or die "Cannot parse cpu description: $cputype\n";
3092         $cpu = $cpuconf->{cputype};
3093         $kvm_off = 1 if $cpuconf->{hidden};
3094     }
3095
3096     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3097
3098     push @$cpuFlags , '-x2apic'
3099         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3100
3101     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3102
3103     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3104
3105     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3106
3107         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3108         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3109     }
3110
3111     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3112
3113     push @$cpuFlags, 'kvm=off' if $kvm_off;
3114
3115     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3116         die "internal error"; # should not happen
3117
3118     push @$cpuFlags, "vendor=${cpu_vendor}"
3119         if $cpu_vendor ne 'default';
3120
3121     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3122
3123     push @$cmd, '-cpu', $cpu;
3124
3125     my $memory = $conf->{memory} || $defaults->{memory};
3126     my $static_memory = 0;
3127     my $dimm_memory = 0;
3128
3129     if ($hotplug_features->{memory}) {
3130         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3131         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3132         $static_memory = $STATICMEM;
3133         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3134         $dimm_memory = $memory - $static_memory;
3135         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3136
3137     } else {
3138
3139         $static_memory = $memory;
3140         push @$cmd, '-m', $static_memory;
3141     }
3142
3143     if ($conf->{numa}) {
3144
3145         my $numa_totalmemory = undef;
3146         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3147             next if !$conf->{"numa$i"};
3148             my $numa = parse_numa($conf->{"numa$i"});
3149             next if !$numa;
3150             # memory
3151             die "missing numa node$i memory value\n" if !$numa->{memory};
3152             my $numa_memory = $numa->{memory};
3153             $numa_totalmemory += $numa_memory;
3154             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3155
3156             # cpus
3157             my $cpulists = $numa->{cpus};
3158             die "missing numa node$i cpus\n" if !defined($cpulists);
3159             my $cpus = join(',', map {
3160                 my ($start, $end) = @$_;
3161                 defined($end) ? "$start-$end" : $start
3162             } @$cpulists);
3163
3164             # hostnodes
3165             my $hostnodelists = $numa->{hostnodes};
3166             if (defined($hostnodelists)) {
3167                 my $hostnodes;
3168                 foreach my $hostnoderange (@$hostnodelists) {
3169                     my ($start, $end) = @$hostnoderange;
3170                     $hostnodes .= ',' if $hostnodes;
3171                     $hostnodes .= $start;
3172                     $hostnodes .= "-$end" if defined($end);
3173                     $end //= $start;
3174                     for (my $i = $start; $i <= $end; ++$i ) {
3175                         die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3176                     }
3177                 }
3178
3179                 # policy
3180                 my $policy = $numa->{policy};
3181                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3182                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3183             }
3184
3185             push @$cmd, '-object', $numa_object;
3186             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3187         }
3188
3189         die "total memory for NUMA nodes must be equal to vm static memory\n"
3190             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3191
3192         #if no custom tology, we split memory and cores across numa nodes
3193         if(!$numa_totalmemory) {
3194
3195             my $numa_memory = ($static_memory / $sockets) . "M";
3196
3197             for (my $i = 0; $i < $sockets; $i++)  {
3198
3199                 my $cpustart = ($cores * $i);
3200                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3201                 my $cpus = $cpustart;
3202                 $cpus .= "-$cpuend" if $cpuend;
3203
3204                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3205                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3206             }
3207         }
3208     }
3209
3210     if ($hotplug_features->{memory}) {
3211         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3212             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3213             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3214             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3215
3216             #if dimm_memory is not aligned to dimm map
3217             if($current_size > $memory) {
3218                  $conf->{memory} = $current_size;
3219                  PVE::QemuConfig->write_config($vmid, $conf);
3220             }
3221         });
3222     }
3223
3224     push @$cmd, '-S' if $conf->{freeze};
3225
3226     # set keyboard layout
3227     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3228     push @$cmd, '-k', $kb if $kb;
3229
3230     # enable sound
3231     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3232     #push @$cmd, '-soundhw', 'es1370';
3233     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3234
3235     if($conf->{agent}) {
3236         my $qgasocket = qmp_socket($vmid, 1);
3237         my $pciaddr = print_pci_addr("qga0", $bridges);
3238         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3239         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3240         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3241     }
3242
3243     my $spice_port;
3244
3245     if ($qxlnum) {
3246         if ($qxlnum > 1) {
3247             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3248                 for(my $i = 1; $i < $qxlnum; $i++){
3249                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3250                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3251                 }
3252             } else {
3253                 # assume other OS works like Linux
3254                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3255                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3256             }
3257         }
3258
3259         my $pciaddr = print_pci_addr("spice", $bridges);
3260
3261         my $nodename = PVE::INotify::nodename();
3262         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3263         $spice_port = PVE::Tools::next_spice_port($pfamily);
3264
3265         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3266
3267         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3268         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3269         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3270     }
3271
3272     # enable balloon by default, unless explicitly disabled
3273     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3274         $pciaddr = print_pci_addr("balloon0", $bridges);
3275         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3276     }
3277
3278     if ($conf->{watchdog}) {
3279         my $wdopts = parse_watchdog($conf->{watchdog});
3280         $pciaddr = print_pci_addr("watchdog", $bridges);
3281         my $watchdog = $wdopts->{model} || 'i6300esb';
3282         push @$devices, '-device', "$watchdog$pciaddr";
3283         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3284     }
3285
3286     my $vollist = [];
3287     my $scsicontroller = {};
3288     my $ahcicontroller = {};
3289     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3290
3291     # Add iscsi initiator name if available
3292     if (my $initiator = get_initiator_name()) {
3293         push @$devices, '-iscsi', "initiator-name=$initiator";
3294     }
3295
3296     foreach_drive($conf, sub {
3297         my ($ds, $drive) = @_;
3298
3299         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3300             push @$vollist, $drive->{file};
3301         }
3302
3303         $use_virtio = 1 if $ds =~ m/^virtio/;
3304
3305         if (drive_is_cdrom ($drive)) {
3306             if ($bootindex_hash->{d}) {
3307                 $drive->{bootindex} = $bootindex_hash->{d};
3308                 $bootindex_hash->{d} += 1;
3309             }
3310         } else {
3311             if ($bootindex_hash->{c}) {
3312                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3313                 $bootindex_hash->{c} += 1;
3314             }
3315         }
3316
3317         if($drive->{interface} eq 'virtio'){
3318            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3319         }
3320
3321         if ($drive->{interface} eq 'scsi') {
3322
3323             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3324
3325             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3326             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3327
3328             my $iothread = '';
3329             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3330                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3331                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3332             }
3333
3334             my $queues = '';
3335             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3336                 $queues = ",num_queues=$drive->{queues}";
3337             }
3338
3339             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3340             $scsicontroller->{$controller}=1;
3341         }
3342
3343         if ($drive->{interface} eq 'sata') {
3344            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3345            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3346            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3347            $ahcicontroller->{$controller}=1;
3348         }
3349
3350         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3351         push @$devices, '-drive',$drive_cmd;
3352         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3353     });
3354
3355     for (my $i = 0; $i < $MAX_NETS; $i++) {
3356          next if !$conf->{"net$i"};
3357          my $d = parse_net($conf->{"net$i"});
3358          next if !$d;
3359
3360          $use_virtio = 1 if $d->{model} eq 'virtio';
3361
3362          if ($bootindex_hash->{n}) {
3363             $d->{bootindex} = $bootindex_hash->{n};
3364             $bootindex_hash->{n} += 1;
3365          }
3366
3367          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3368          push @$devices, '-netdev', $netdevfull;
3369
3370          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3371          push @$devices, '-device', $netdevicefull;
3372     }
3373
3374     if (!$q35) {
3375         # add pci bridges
3376         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3377            $bridges->{1} = 1;
3378            $bridges->{2} = 1;
3379         }
3380
3381         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3382
3383         while (my ($k, $v) = each %$bridges) {
3384             $pciaddr = print_pci_addr("pci.$k");
3385             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3386         }
3387     }
3388
3389     # add custom args
3390     if ($conf->{args}) {
3391         my $aa = PVE::Tools::split_args($conf->{args});
3392         push @$cmd, @$aa;
3393     }
3394
3395     push @$cmd, @$devices;
3396     push @$cmd, '-rtc', join(',', @$rtcFlags)
3397         if scalar(@$rtcFlags);
3398     push @$cmd, '-machine', join(',', @$machineFlags)
3399         if scalar(@$machineFlags);
3400     push @$cmd, '-global', join(',', @$globalFlags)
3401         if scalar(@$globalFlags);
3402
3403     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3404 }
3405
3406 sub vnc_socket {
3407     my ($vmid) = @_;
3408     return "${var_run_tmpdir}/$vmid.vnc";
3409 }
3410
3411 sub spice_port {
3412     my ($vmid) = @_;
3413
3414     my $res = vm_mon_cmd($vmid, 'query-spice');
3415
3416     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3417 }
3418
3419 sub qmp_socket {
3420     my ($vmid, $qga) = @_;
3421     my $sockettype = $qga ? 'qga' : 'qmp';
3422     return "${var_run_tmpdir}/$vmid.$sockettype";
3423 }
3424
3425 sub pidfile_name {
3426     my ($vmid) = @_;
3427     return "${var_run_tmpdir}/$vmid.pid";
3428 }
3429
3430 sub vm_devices_list {
3431     my ($vmid) = @_;
3432
3433     my $res = vm_mon_cmd($vmid, 'query-pci');
3434     my $devices = {};
3435     foreach my $pcibus (@$res) {
3436         foreach my $device (@{$pcibus->{devices}}) {
3437             next if !$device->{'qdev_id'};
3438             if ($device->{'pci_bridge'}) {
3439                 $devices->{$device->{'qdev_id'}} = 1;
3440                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3441                     next if !$bridge_device->{'qdev_id'};
3442                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3443                     $devices->{$device->{'qdev_id'}}++;
3444                 }
3445             } else {
3446                 $devices->{$device->{'qdev_id'}} = 1;
3447             }
3448         }
3449     }
3450
3451     my $resblock = vm_mon_cmd($vmid, 'query-block');
3452     foreach my $block (@$resblock) {
3453         if($block->{device} =~ m/^drive-(\S+)/){
3454                 $devices->{$1} = 1;
3455         }
3456     }
3457
3458     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3459     foreach my $mice (@$resmice) {
3460         if ($mice->{name} eq 'QEMU HID Tablet') {
3461             $devices->{tablet} = 1;
3462             last;
3463         }
3464     }
3465
3466     return $devices;
3467 }
3468
3469 sub vm_deviceplug {
3470     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3471
3472     my $q35 = machine_type_is_q35($conf);
3473
3474     my $devices_list = vm_devices_list($vmid);
3475     return 1 if defined($devices_list->{$deviceid});
3476
3477     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3478
3479     if ($deviceid eq 'tablet') {
3480
3481         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3482
3483     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3484
3485         qemu_iothread_add($vmid, $deviceid, $device);
3486
3487         qemu_driveadd($storecfg, $vmid, $device);
3488         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3489
3490         qemu_deviceadd($vmid, $devicefull);
3491         eval { qemu_deviceaddverify($vmid, $deviceid); };
3492         if (my $err = $@) {
3493             eval { qemu_drivedel($vmid, $deviceid); };
3494             warn $@ if $@;
3495             die $err;
3496         }
3497
3498     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3499
3500
3501         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3502         my $pciaddr = print_pci_addr($deviceid);
3503         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3504
3505         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3506
3507         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3508             qemu_iothread_add($vmid, $deviceid, $device);
3509             $devicefull .= ",iothread=iothread-$deviceid";
3510         }
3511
3512         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3513             $devicefull .= ",num_queues=$device->{queues}";
3514         }
3515
3516         qemu_deviceadd($vmid, $devicefull);
3517         qemu_deviceaddverify($vmid, $deviceid);
3518
3519     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3520
3521         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3522         qemu_driveadd($storecfg, $vmid, $device);
3523
3524         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3525         eval { qemu_deviceadd($vmid, $devicefull); };
3526         if (my $err = $@) {
3527             eval { qemu_drivedel($vmid, $deviceid); };
3528             warn $@ if $@;
3529             die $err;
3530         }
3531
3532     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3533
3534         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3535
3536         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3537         my $use_old_bios_files = undef;
3538         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3539
3540         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3541         qemu_deviceadd($vmid, $netdevicefull);
3542         eval { qemu_deviceaddverify($vmid, $deviceid); };
3543         if (my $err = $@) {
3544             eval { qemu_netdevdel($vmid, $deviceid); };
3545             warn $@ if $@;
3546             die $err;
3547         }
3548
3549     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3550
3551         my $bridgeid = $2;
3552         my $pciaddr = print_pci_addr($deviceid);
3553         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3554
3555         qemu_deviceadd($vmid, $devicefull);
3556         qemu_deviceaddverify($vmid, $deviceid);
3557
3558     } else {
3559         die "can't hotplug device '$deviceid'\n";
3560     }
3561
3562     return 1;
3563 }
3564
3565 # fixme: this should raise exceptions on error!
3566 sub vm_deviceunplug {
3567     my ($vmid, $conf, $deviceid) = @_;
3568
3569     my $devices_list = vm_devices_list($vmid);
3570     return 1 if !defined($devices_list->{$deviceid});
3571
3572     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3573
3574     if ($deviceid eq 'tablet') {
3575
3576         qemu_devicedel($vmid, $deviceid);
3577
3578     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3579
3580         qemu_devicedel($vmid, $deviceid);
3581         qemu_devicedelverify($vmid, $deviceid);
3582         qemu_drivedel($vmid, $deviceid);
3583         qemu_iothread_del($conf, $vmid, $deviceid);
3584
3585     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3586
3587         qemu_devicedel($vmid, $deviceid);
3588         qemu_devicedelverify($vmid, $deviceid);
3589         qemu_iothread_del($conf, $vmid, $deviceid);
3590
3591     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3592
3593         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3594         my $device = parse_drive($deviceid, $conf->{$deviceid});
3595         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3596
3597         qemu_devicedel($vmid, $deviceid);
3598         qemu_drivedel($vmid, $deviceid);
3599         qemu_deletescsihw($conf, $vmid, $deviceid);
3600
3601     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3602
3603         qemu_devicedel($vmid, $deviceid);
3604         qemu_devicedelverify($vmid, $deviceid);
3605         qemu_netdevdel($vmid, $deviceid);
3606
3607     } else {
3608         die "can't unplug device '$deviceid'\n";
3609     }
3610
3611     return 1;
3612 }
3613
3614 sub qemu_deviceadd {
3615     my ($vmid, $devicefull) = @_;
3616
3617     $devicefull = "driver=".$devicefull;
3618     my %options =  split(/[=,]/, $devicefull);
3619
3620     vm_mon_cmd($vmid, "device_add" , %options);
3621 }
3622
3623 sub qemu_devicedel {
3624     my ($vmid, $deviceid) = @_;
3625
3626     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3627 }
3628
3629 sub qemu_iothread_add {
3630     my($vmid, $deviceid, $device) = @_;
3631
3632     if ($device->{iothread}) {
3633         my $iothreads = vm_iothreads_list($vmid);
3634         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3635     }
3636 }
3637
3638 sub qemu_iothread_del {
3639     my($conf, $vmid, $deviceid) = @_;
3640
3641     my $device = parse_drive($deviceid, $conf->{$deviceid});
3642     if ($device->{iothread}) {
3643         my $iothreads = vm_iothreads_list($vmid);
3644         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3645     }
3646 }
3647
3648 sub qemu_objectadd {
3649     my($vmid, $objectid, $qomtype) = @_;
3650
3651     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3652
3653     return 1;
3654 }
3655
3656 sub qemu_objectdel {
3657     my($vmid, $objectid) = @_;
3658
3659     vm_mon_cmd($vmid, "object-del", id => $objectid);
3660
3661     return 1;
3662 }
3663
3664 sub qemu_driveadd {
3665     my ($storecfg, $vmid, $device) = @_;
3666
3667     my $drive = print_drive_full($storecfg, $vmid, $device);
3668     $drive =~ s/\\/\\\\/g;
3669     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3670
3671     # If the command succeeds qemu prints: "OK"
3672     return 1 if $ret =~ m/OK/s;
3673
3674     die "adding drive failed: $ret\n";
3675 }
3676
3677 sub qemu_drivedel {
3678     my($vmid, $deviceid) = @_;
3679
3680     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3681     $ret =~ s/^\s+//;
3682
3683     return 1 if $ret eq "";
3684
3685     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3686     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3687
3688     die "deleting drive $deviceid failed : $ret\n";
3689 }
3690
3691 sub qemu_deviceaddverify {
3692     my ($vmid, $deviceid) = @_;
3693
3694     for (my $i = 0; $i <= 5; $i++) {
3695          my $devices_list = vm_devices_list($vmid);
3696          return 1 if defined($devices_list->{$deviceid});
3697          sleep 1;
3698     }
3699
3700     die "error on hotplug device '$deviceid'\n";
3701 }
3702
3703
3704 sub qemu_devicedelverify {
3705     my ($vmid, $deviceid) = @_;
3706
3707     # need to verify that the device is correctly removed as device_del
3708     # is async and empty return is not reliable
3709
3710     for (my $i = 0; $i <= 5; $i++) {
3711          my $devices_list = vm_devices_list($vmid);
3712          return 1 if !defined($devices_list->{$deviceid});
3713          sleep 1;
3714     }
3715
3716     die "error on hot-unplugging device '$deviceid'\n";
3717 }
3718
3719 sub qemu_findorcreatescsihw {
3720     my ($storecfg, $conf, $vmid, $device) = @_;
3721
3722     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3723
3724     my $scsihwid="$controller_prefix$controller";
3725     my $devices_list = vm_devices_list($vmid);
3726
3727     if(!defined($devices_list->{$scsihwid})) {
3728         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3729     }
3730
3731     return 1;
3732 }
3733
3734 sub qemu_deletescsihw {
3735     my ($conf, $vmid, $opt) = @_;
3736
3737     my $device = parse_drive($opt, $conf->{$opt});
3738
3739     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3740         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3741         return 1;
3742     }
3743
3744     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3745
3746     my $devices_list = vm_devices_list($vmid);
3747     foreach my $opt (keys %{$devices_list}) {
3748         if (PVE::QemuServer::is_valid_drivename($opt)) {
3749             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3750             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3751                 return 1;
3752             }
3753         }
3754     }
3755
3756     my $scsihwid="scsihw$controller";
3757
3758     vm_deviceunplug($vmid, $conf, $scsihwid);
3759
3760     return 1;
3761 }
3762
3763 sub qemu_add_pci_bridge {
3764     my ($storecfg, $conf, $vmid, $device) = @_;
3765
3766     my $bridges = {};
3767
3768     my $bridgeid;
3769
3770     print_pci_addr($device, $bridges);
3771
3772     while (my ($k, $v) = each %$bridges) {
3773         $bridgeid = $k;
3774     }
3775     return 1 if !defined($bridgeid) || $bridgeid < 1;
3776
3777     my $bridge = "pci.$bridgeid";
3778     my $devices_list = vm_devices_list($vmid);
3779
3780     if (!defined($devices_list->{$bridge})) {
3781         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3782     }
3783
3784     return 1;
3785 }
3786
3787 sub qemu_set_link_status {
3788     my ($vmid, $device, $up) = @_;
3789
3790     vm_mon_cmd($vmid, "set_link", name => $device,
3791                up => $up ? JSON::true : JSON::false);
3792 }
3793
3794 sub qemu_netdevadd {
3795     my ($vmid, $conf, $device, $deviceid) = @_;
3796
3797     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3798     my %options =  split(/[=,]/, $netdev);
3799
3800     vm_mon_cmd($vmid, "netdev_add",  %options);
3801     return 1;
3802 }
3803
3804 sub qemu_netdevdel {
3805     my ($vmid, $deviceid) = @_;
3806
3807     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3808 }
3809
3810 sub qemu_cpu_hotplug {
3811     my ($vmid, $conf, $vcpus) = @_;
3812
3813     my $sockets = 1;
3814     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3815     $sockets = $conf->{sockets} if  $conf->{sockets};
3816     my $cores = $conf->{cores} || 1;
3817     my $maxcpus = $sockets * $cores;
3818
3819     $vcpus = $maxcpus if !$vcpus;
3820
3821     die "you can't add more vcpus than maxcpus\n"
3822         if $vcpus > $maxcpus;
3823
3824     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3825     die "online cpu unplug is not yet possible\n"
3826         if $vcpus < $currentvcpus;
3827
3828     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3829     die "vcpus in running vm is different than configuration\n"
3830         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3831
3832     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3833         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3834     }
3835 }
3836
3837 sub qemu_memory_hotplug {
3838     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3839
3840     return $value if !check_running($vmid);
3841
3842     my $memory = $conf->{memory} || $defaults->{memory};
3843     $value = $defaults->{memory} if !$value;
3844     return $value if $value == $memory;
3845
3846     my $static_memory = $STATICMEM;
3847     my $dimm_memory = $memory - $static_memory;
3848
3849     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3850     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3851
3852
3853     my $sockets = 1;
3854     $sockets = $conf->{sockets} if $conf->{sockets};
3855
3856     if($value > $memory) {
3857
3858         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3859             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3860
3861                 return if $current_size <= $conf->{memory};
3862
3863                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3864                 if (my $err = $@) {
3865                     eval { qemu_objectdel($vmid, "mem-$name"); };
3866                     die $err;
3867                 }
3868
3869                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3870                 if (my $err = $@) {
3871                     eval { qemu_objectdel($vmid, "mem-$name"); };
3872                     die $err;
3873                 }
3874                 #update conf after each succesful module hotplug
3875                 $conf->{memory} = $current_size;
3876                 PVE::QemuConfig->write_config($vmid, $conf);
3877         });
3878
3879     } else {
3880
3881         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3882             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3883
3884                 return if $current_size >= $conf->{memory};
3885                 print "try to unplug memory dimm $name\n";
3886
3887                 my $retry = 0;
3888                 while (1) {
3889                     eval { qemu_devicedel($vmid, $name) };
3890                     sleep 3;
3891                     my $dimm_list = qemu_dimm_list($vmid);
3892                     last if !$dimm_list->{$name};
3893                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3894                     $retry++;
3895                 }
3896
3897                 #update conf after each succesful module unplug
3898                 $conf->{memory} = $current_size;
3899
3900                 eval { qemu_objectdel($vmid, "mem-$name"); };
3901                 PVE::QemuConfig->write_config($vmid, $conf);
3902         });
3903     }
3904 }
3905
3906 sub qemu_dimm_list {
3907     my ($vmid) = @_;
3908
3909     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3910     my $dimms = {};
3911
3912     foreach my $dimm (@$dimmarray) {
3913
3914         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3915         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3916         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3917         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3918         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3919     }
3920     return $dimms;
3921 }
3922
3923 sub qemu_block_set_io_throttle {
3924     my ($vmid, $deviceid,
3925         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3926         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3927
3928     return if !check_running($vmid) ;
3929
3930     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3931         bps => int($bps),
3932         bps_rd => int($bps_rd),
3933         bps_wr => int($bps_wr),
3934         iops => int($iops),
3935         iops_rd => int($iops_rd),
3936         iops_wr => int($iops_wr),
3937         bps_max => int($bps_max),
3938         bps_rd_max => int($bps_rd_max),
3939         bps_wr_max => int($bps_wr_max),
3940         iops_max => int($iops_max),
3941         iops_rd_max => int($iops_rd_max),
3942         iops_wr_max => int($iops_wr_max)
3943     );
3944
3945 }
3946
3947 # old code, only used to shutdown old VM after update
3948 sub __read_avail {
3949     my ($fh, $timeout) = @_;
3950
3951     my $sel = new IO::Select;
3952     $sel->add($fh);
3953
3954     my $res = '';
3955     my $buf;
3956
3957     my @ready;
3958     while (scalar (@ready = $sel->can_read($timeout))) {
3959         my $count;
3960         if ($count = $fh->sysread($buf, 8192)) {
3961             if ($buf =~ /^(.*)\(qemu\) $/s) {
3962                 $res .= $1;
3963                 last;
3964             } else {
3965                 $res .= $buf;
3966             }
3967         } else {
3968             if (!defined($count)) {
3969                 die "$!\n";
3970             }
3971             last;
3972         }
3973     }
3974
3975     die "monitor read timeout\n" if !scalar(@ready);
3976
3977     return $res;
3978 }
3979
3980 # old code, only used to shutdown old VM after update
3981 sub vm_monitor_command {
3982     my ($vmid, $cmdstr, $nocheck) = @_;
3983
3984     my $res;
3985
3986     eval {
3987         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3988
3989         my $sname = "${var_run_tmpdir}/$vmid.mon";
3990
3991         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3992             die "unable to connect to VM $vmid socket - $!\n";
3993
3994         my $timeout = 3;
3995
3996         # hack: migrate sometime blocks the monitor (when migrate_downtime
3997         # is set)
3998         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3999             $timeout = 60*60; # 1 hour
4000         }
4001
4002         # read banner;
4003         my $data = __read_avail($sock, $timeout);
4004
4005         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4006             die "got unexpected qemu monitor banner\n";
4007         }
4008
4009         my $sel = new IO::Select;
4010         $sel->add($sock);
4011
4012         if (!scalar(my @ready = $sel->can_write($timeout))) {
4013             die "monitor write error - timeout";
4014         }
4015
4016         my $fullcmd = "$cmdstr\r";
4017
4018         # syslog('info', "VM $vmid monitor command: $cmdstr");
4019
4020         my $b;
4021         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4022             die "monitor write error - $!";
4023         }
4024
4025         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4026
4027         $timeout = 20;
4028
4029         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4030             $timeout = 60*60; # 1 hour
4031         } elsif ($cmdstr =~ m/^(eject|change)/) {
4032             $timeout = 60; # note: cdrom mount command is slow
4033         }
4034         if ($res = __read_avail($sock, $timeout)) {
4035
4036             my @lines = split("\r?\n", $res);
4037
4038             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4039
4040             $res = join("\n", @lines);
4041             $res .= "\n";
4042         }
4043     };
4044
4045     my $err = $@;
4046
4047     if ($err) {
4048         syslog("err", "VM $vmid monitor command failed - $err");
4049         die $err;
4050     }
4051
4052     return $res;
4053 }
4054
4055 sub qemu_block_resize {
4056     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4057
4058     my $running = check_running($vmid);
4059
4060     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4061
4062     return if !$running;
4063
4064     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4065
4066 }
4067
4068 sub qemu_volume_snapshot {
4069     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4070
4071     my $running = check_running($vmid);
4072
4073     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4074         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4075     } else {
4076         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4077     }
4078 }
4079
4080 sub qemu_volume_snapshot_delete {
4081     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4082
4083     my $running = check_running($vmid);
4084
4085     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4086
4087     return if !$running;
4088
4089     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4090 }
4091
4092 sub set_migration_caps {
4093     my ($vmid) = @_;
4094
4095     my $cap_ref = [];
4096
4097     my $enabled_cap = {
4098         "auto-converge" => 1,
4099         "xbzrle" => 1,
4100         "x-rdma-pin-all" => 0,
4101         "zero-blocks" => 0,
4102         "compress" => 0
4103     };
4104
4105     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4106
4107     for my $supported_capability (@$supported_capabilities) {
4108         push @$cap_ref, {
4109             capability => $supported_capability->{capability},
4110             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4111         };
4112     }
4113
4114     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4115 }
4116
4117 my $fast_plug_option = {
4118     'lock' => 1,
4119     'name' => 1,
4120     'onboot' => 1,
4121     'shares' => 1,
4122     'startup' => 1,
4123     'description' => 1,
4124 };
4125
4126 # hotplug changes in [PENDING]
4127 # $selection hash can be used to only apply specified options, for
4128 # example: { cores => 1 } (only apply changed 'cores')
4129 # $errors ref is used to return error messages
4130 sub vmconfig_hotplug_pending {
4131     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4132
4133     my $defaults = load_defaults();
4134
4135     # commit values which do not have any impact on running VM first
4136     # Note: those option cannot raise errors, we we do not care about
4137     # $selection and always apply them.
4138
4139     my $add_error = sub {
4140         my ($opt, $msg) = @_;
4141         $errors->{$opt} = "hotplug problem - $msg";
4142     };
4143
4144     my $changes = 0;
4145     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4146         if ($fast_plug_option->{$opt}) {
4147             $conf->{$opt} = $conf->{pending}->{$opt};
4148             delete $conf->{pending}->{$opt};
4149             $changes = 1;
4150         }
4151     }
4152
4153     if ($changes) {
4154         PVE::QemuConfig->write_config($vmid, $conf);
4155         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4156     }
4157
4158     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4159
4160     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4161     while (my ($opt, $force) = each %$pending_delete_hash) {
4162         next if $selection && !$selection->{$opt};
4163         eval {
4164             if ($opt eq 'hotplug') {
4165                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4166             } elsif ($opt eq 'tablet') {
4167                 die "skip\n" if !$hotplug_features->{usb};
4168                 if ($defaults->{tablet}) {
4169                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4170                 } else {
4171                     vm_deviceunplug($vmid, $conf, $opt);
4172                 }
4173             } elsif ($opt eq 'vcpus') {
4174                 die "skip\n" if !$hotplug_features->{cpu};
4175                 qemu_cpu_hotplug($vmid, $conf, undef);
4176             } elsif ($opt eq 'balloon') {
4177                 # enable balloon device is not hotpluggable
4178                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4179             } elsif ($fast_plug_option->{$opt}) {
4180                 # do nothing
4181             } elsif ($opt =~ m/^net(\d+)$/) {
4182                 die "skip\n" if !$hotplug_features->{network};
4183                 vm_deviceunplug($vmid, $conf, $opt);
4184             } elsif (is_valid_drivename($opt)) {
4185                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4186                 vm_deviceunplug($vmid, $conf, $opt);
4187                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4188             } elsif ($opt =~ m/^memory$/) {
4189                 die "skip\n" if !$hotplug_features->{memory};
4190                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4191             } elsif ($opt eq 'cpuunits') {
4192                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4193             } elsif ($opt eq 'cpulimit') {
4194                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4195             } else {
4196                 die "skip\n";
4197             }
4198         };
4199         if (my $err = $@) {
4200             &$add_error($opt, $err) if $err ne "skip\n";
4201         } else {
4202             # save new config if hotplug was successful
4203             delete $conf->{$opt};
4204             vmconfig_undelete_pending_option($conf, $opt);
4205             PVE::QemuConfig->write_config($vmid, $conf);
4206             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4207         }
4208     }
4209
4210     foreach my $opt (keys %{$conf->{pending}}) {
4211         next if $selection && !$selection->{$opt};
4212         my $value = $conf->{pending}->{$opt};
4213         eval {
4214             if ($opt eq 'hotplug') {
4215                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4216             } elsif ($opt eq 'tablet') {
4217                 die "skip\n" if !$hotplug_features->{usb};
4218                 if ($value == 1) {
4219                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4220                 } elsif ($value == 0) {
4221                     vm_deviceunplug($vmid, $conf, $opt);
4222                 }
4223             } elsif ($opt eq 'vcpus') {
4224                 die "skip\n" if !$hotplug_features->{cpu};
4225                 qemu_cpu_hotplug($vmid, $conf, $value);
4226             } elsif ($opt eq 'balloon') {
4227                 # enable/disable balloning device is not hotpluggable
4228                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4229                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4230                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4231
4232                 # allow manual ballooning if shares is set to zero
4233                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4234                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4235                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4236                 }
4237             } elsif ($opt =~ m/^net(\d+)$/) {
4238                 # some changes can be done without hotplug
4239                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4240                                     $vmid, $opt, $value);
4241             } elsif (is_valid_drivename($opt)) {
4242                 # some changes can be done without hotplug
4243                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4244                                      $vmid, $opt, $value, 1);
4245             } elsif ($opt =~ m/^memory$/) { #dimms
4246                 die "skip\n" if !$hotplug_features->{memory};
4247                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4248             } elsif ($opt eq 'cpuunits') {
4249                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4250             } elsif ($opt eq 'cpulimit') {
4251                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4252                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4253             } else {
4254                 die "skip\n";  # skip non-hot-pluggable options
4255             }
4256         };
4257         if (my $err = $@) {
4258             &$add_error($opt, $err) if $err ne "skip\n";
4259         } else {
4260             # save new config if hotplug was successful
4261             $conf->{$opt} = $value;
4262             delete $conf->{pending}->{$opt};
4263             PVE::QemuConfig->write_config($vmid, $conf);
4264             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4265         }
4266     }
4267 }
4268
4269 sub try_deallocate_drive {
4270     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4271
4272     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4273         my $volid = $drive->{file};
4274         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4275             my $sid = PVE::Storage::parse_volume_id($volid);
4276             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4277
4278             # check if the disk is really unused
4279             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4280                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4281             PVE::Storage::vdisk_free($storecfg, $volid);
4282             return 1;
4283         } else {
4284             # If vm is not owner of this disk remove from config
4285             return 1;
4286         }
4287     }
4288
4289     return undef;
4290 }
4291
4292 sub vmconfig_delete_or_detach_drive {
4293     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4294
4295     my $drive = parse_drive($opt, $conf->{$opt});
4296
4297     my $rpcenv = PVE::RPCEnvironment::get();
4298     my $authuser = $rpcenv->get_user();
4299
4300     if ($force) {
4301         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4302         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4303     } else {
4304         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4305     }
4306 }
4307
4308 sub vmconfig_apply_pending {
4309     my ($vmid, $conf, $storecfg) = @_;
4310
4311     # cold plug
4312
4313     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4314     while (my ($opt, $force) = each %$pending_delete_hash) {
4315         die "internal error" if $opt =~ m/^unused/;
4316         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4317         if (!defined($conf->{$opt})) {
4318             vmconfig_undelete_pending_option($conf, $opt);
4319             PVE::QemuConfig->write_config($vmid, $conf);
4320         } elsif (is_valid_drivename($opt)) {
4321             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4322             vmconfig_undelete_pending_option($conf, $opt);
4323             delete $conf->{$opt};
4324             PVE::QemuConfig->write_config($vmid, $conf);
4325         } else {
4326             vmconfig_undelete_pending_option($conf, $opt);
4327             delete $conf->{$opt};
4328             PVE::QemuConfig->write_config($vmid, $conf);
4329         }
4330     }
4331
4332     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4333
4334     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4335         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4336
4337         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4338             # skip if nothing changed
4339         } elsif (is_valid_drivename($opt)) {
4340             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4341                 if defined($conf->{$opt});
4342             $conf->{$opt} = $conf->{pending}->{$opt};
4343         } else {
4344             $conf->{$opt} = $conf->{pending}->{$opt};
4345         }
4346
4347         delete $conf->{pending}->{$opt};
4348         PVE::QemuConfig->write_config($vmid, $conf);
4349     }
4350 }
4351
4352 my $safe_num_ne = sub {
4353     my ($a, $b) = @_;
4354
4355     return 0 if !defined($a) && !defined($b);
4356     return 1 if !defined($a);
4357     return 1 if !defined($b);
4358
4359     return $a != $b;
4360 };
4361
4362 my $safe_string_ne = sub {
4363     my ($a, $b) = @_;
4364
4365     return 0 if !defined($a) && !defined($b);
4366     return 1 if !defined($a);
4367     return 1 if !defined($b);
4368
4369     return $a ne $b;
4370 };
4371
4372 sub vmconfig_update_net {
4373     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4374
4375     my $newnet = parse_net($value);
4376
4377     if ($conf->{$opt}) {
4378         my $oldnet = parse_net($conf->{$opt});
4379
4380         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4381             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4382             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4383             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4384
4385             # for non online change, we try to hot-unplug
4386             die "skip\n" if !$hotplug;
4387             vm_deviceunplug($vmid, $conf, $opt);
4388         } else {
4389
4390             die "internal error" if $opt !~ m/net(\d+)/;
4391             my $iface = "tap${vmid}i$1";
4392
4393             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4394                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4395                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4396                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4397                 PVE::Network::tap_unplug($iface);
4398                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4399             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4400                 # Rate can be applied on its own but any change above needs to
4401                 # include the rate in tap_plug since OVS resets everything.
4402                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4403             }
4404
4405             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4406                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4407             }
4408
4409             return 1;
4410         }
4411     }
4412
4413     if ($hotplug) {
4414         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4415     } else {
4416         die "skip\n";
4417     }
4418 }
4419
4420 sub vmconfig_update_disk {
4421     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4422
4423     # fixme: do we need force?
4424
4425     my $drive = parse_drive($opt, $value);
4426
4427     if ($conf->{$opt}) {
4428
4429         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4430
4431             my $media = $drive->{media} || 'disk';
4432             my $oldmedia = $old_drive->{media} || 'disk';
4433             die "unable to change media type\n" if $media ne $oldmedia;
4434
4435             if (!drive_is_cdrom($old_drive)) {
4436
4437                 if ($drive->{file} ne $old_drive->{file}) {
4438
4439                     die "skip\n" if !$hotplug;
4440
4441                     # unplug and register as unused
4442                     vm_deviceunplug($vmid, $conf, $opt);
4443                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4444
4445                 } else {
4446                     # update existing disk
4447
4448                     # skip non hotpluggable value
4449                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4450                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4451                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4452                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4453                         die "skip\n";
4454                     }
4455
4456                     # apply throttle
4457                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4458                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4459                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4460                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4461                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4462                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4463                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4464                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4465                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4466                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4467                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4468                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4469
4470                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4471                                                    ($drive->{mbps} || 0)*1024*1024,
4472                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4473                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4474                                                    $drive->{iops} || 0,
4475                                                    $drive->{iops_rd} || 0,
4476                                                    $drive->{iops_wr} || 0,
4477                                                    ($drive->{mbps_max} || 0)*1024*1024,
4478                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4479                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4480                                                    $drive->{iops_max} || 0,
4481                                                    $drive->{iops_rd_max} || 0,
4482                                                    $drive->{iops_wr_max} || 0);
4483
4484                     }
4485
4486                     return 1;
4487                 }
4488
4489             } else { # cdrom
4490
4491                 if ($drive->{file} eq 'none') {
4492                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4493                 } else {
4494                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4495                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4496                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4497                 }
4498
4499                 return 1;
4500             }
4501         }
4502     }
4503
4504     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4505     # hotplug new disks
4506     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4507     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4508 }
4509
4510 sub vm_start {
4511     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4512         $forcemachine, $spice_ticket) = @_;
4513
4514     PVE::QemuConfig->lock_config($vmid, sub {
4515         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4516
4517         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4518
4519         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4520
4521         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4522
4523         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4524             vmconfig_apply_pending($vmid, $conf, $storecfg);
4525             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4526         }
4527
4528         my $defaults = load_defaults();
4529
4530         # set environment variable useful inside network script
4531         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4532
4533         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4534
4535         my $migrate_port = 0;
4536         my $migrate_uri;
4537         if ($statefile) {
4538             if ($statefile eq 'tcp') {
4539                 my $localip = "localhost";
4540                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4541                 my $nodename = PVE::INotify::nodename();
4542                 if ($datacenterconf->{migration_unsecure}) {
4543                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4544                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4545                 }
4546                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4547                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4548                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4549                 push @$cmd, '-incoming', $migrate_uri;
4550                 push @$cmd, '-S';
4551             } else {
4552                 push @$cmd, '-loadstate', $statefile;
4553             }
4554         } elsif ($paused) {
4555             push @$cmd, '-S';
4556         }
4557
4558         # host pci devices
4559         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4560           my $d = parse_hostpci($conf->{"hostpci$i"});
4561           next if !$d;
4562           my $pcidevices = $d->{pciid};
4563           foreach my $pcidevice (@$pcidevices) {
4564                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4565
4566                 my $info = pci_device_info("0000:$pciid");
4567                 die "IOMMU not present\n" if !check_iommu_support();
4568                 die "no pci device info for device '$pciid'\n" if !$info;
4569                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4570                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4571           }
4572         }
4573
4574         PVE::Storage::activate_volumes($storecfg, $vollist);
4575
4576         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4577             my $cmd = [];
4578             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4579             eval  { run_command($cmd); };
4580         }
4581
4582         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4583                     umask => 0077); };
4584
4585         if (my $err = $@) {
4586             # deactivate volumes if start fails
4587             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4588             die "start failed: $err";
4589         }
4590
4591         print "migration listens on $migrate_uri\n" if $migrate_uri;
4592
4593         if ($statefile && $statefile ne 'tcp')  {
4594             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4595             warn $@ if $@;
4596         }
4597
4598         if ($migratedfrom) {
4599
4600             eval {
4601                 set_migration_caps($vmid);
4602             };
4603             warn $@ if $@;
4604
4605             if ($spice_port) {
4606                 print "spice listens on port $spice_port\n";
4607                 if ($spice_ticket) {
4608                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4609                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4610                 }
4611             }
4612
4613         } else {
4614
4615             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4616                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4617                     if $conf->{balloon};
4618             }
4619
4620             foreach my $opt (keys %$conf) {
4621                 next if $opt !~  m/^net\d+$/;
4622                 my $nicconf = parse_net($conf->{$opt});
4623                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4624             }
4625         }
4626
4627         vm_mon_cmd_nocheck($vmid, 'qom-set',
4628                     path => "machine/peripheral/balloon0",
4629                     property => "guest-stats-polling-interval",
4630                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4631
4632     });
4633 }
4634
4635 sub vm_mon_cmd {
4636     my ($vmid, $execute, %params) = @_;
4637
4638     my $cmd = { execute => $execute, arguments => \%params };
4639     vm_qmp_command($vmid, $cmd);
4640 }
4641
4642 sub vm_mon_cmd_nocheck {
4643     my ($vmid, $execute, %params) = @_;
4644
4645     my $cmd = { execute => $execute, arguments => \%params };
4646     vm_qmp_command($vmid, $cmd, 1);
4647 }
4648
4649 sub vm_qmp_command {
4650     my ($vmid, $cmd, $nocheck) = @_;
4651
4652     my $res;
4653
4654     my $timeout;
4655     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4656         $timeout = $cmd->{arguments}->{timeout};
4657         delete $cmd->{arguments}->{timeout};
4658     }
4659
4660     eval {
4661         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4662         my $sname = qmp_socket($vmid);
4663         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4664             my $qmpclient = PVE::QMPClient->new();
4665
4666             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4667         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4668             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4669                 if scalar(%{$cmd->{arguments}});
4670             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4671         } else {
4672             die "unable to open monitor socket\n";
4673         }
4674     };
4675     if (my $err = $@) {
4676         syslog("err", "VM $vmid qmp command failed - $err");
4677         die $err;
4678     }
4679
4680     return $res;
4681 }
4682
4683 sub vm_human_monitor_command {
4684     my ($vmid, $cmdline) = @_;
4685
4686     my $res;
4687
4688     my $cmd = {
4689         execute => 'human-monitor-command',
4690         arguments => { 'command-line' => $cmdline},
4691     };
4692
4693     return vm_qmp_command($vmid, $cmd);
4694 }
4695
4696 sub vm_commandline {
4697     my ($storecfg, $vmid) = @_;
4698
4699     my $conf = PVE::QemuConfig->load_config($vmid);
4700
4701     my $defaults = load_defaults();
4702
4703     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4704
4705     return join(' ', @$cmd);
4706 }
4707
4708 sub vm_reset {
4709     my ($vmid, $skiplock) = @_;
4710
4711     PVE::QemuConfig->lock_config($vmid, sub {
4712
4713         my $conf = PVE::QemuConfig->load_config($vmid);
4714
4715         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4716
4717         vm_mon_cmd($vmid, "system_reset");
4718     });
4719 }
4720
4721 sub get_vm_volumes {
4722     my ($conf) = @_;
4723
4724     my $vollist = [];
4725     foreach_volid($conf, sub {
4726         my ($volid, $is_cdrom) = @_;
4727
4728         return if $volid =~ m|^/|;
4729
4730         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4731         return if !$sid;
4732
4733         push @$vollist, $volid;
4734     });
4735
4736     return $vollist;
4737 }
4738
4739 sub vm_stop_cleanup {
4740     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4741
4742     eval {
4743
4744         if (!$keepActive) {
4745             my $vollist = get_vm_volumes($conf);
4746             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4747         }
4748
4749         foreach my $ext (qw(mon qmp pid vnc qga)) {
4750             unlink "/var/run/qemu-server/${vmid}.$ext";
4751         }
4752
4753         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4754     };
4755     warn $@ if $@; # avoid errors - just warn
4756 }
4757
4758 # Note: use $nockeck to skip tests if VM configuration file exists.
4759 # We need that when migration VMs to other nodes (files already moved)
4760 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4761 sub vm_stop {
4762     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4763
4764     $force = 1 if !defined($force) && !$shutdown;
4765
4766     if ($migratedfrom){
4767         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4768         kill 15, $pid if $pid;
4769         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4770         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4771         return;
4772     }
4773
4774     PVE::QemuConfig->lock_config($vmid, sub {
4775
4776         my $pid = check_running($vmid, $nocheck);
4777         return if !$pid;
4778
4779         my $conf;
4780         if (!$nocheck) {
4781             $conf = PVE::QemuConfig->load_config($vmid);
4782             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4783             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4784                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4785                 $timeout = $opts->{down} if $opts->{down};
4786             }
4787         }
4788
4789         $timeout = 60 if !defined($timeout);
4790
4791         eval {
4792             if ($shutdown) {
4793                 if (defined($conf) && $conf->{agent}) {
4794                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4795                 } else {
4796                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4797                 }
4798             } else {
4799                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4800             }
4801         };
4802         my $err = $@;
4803
4804         if (!$err) {
4805             my $count = 0;
4806             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4807                 $count++;
4808                 sleep 1;
4809             }
4810
4811             if ($count >= $timeout) {
4812                 if ($force) {
4813                     warn "VM still running - terminating now with SIGTERM\n";
4814                     kill 15, $pid;
4815                 } else {
4816                     die "VM quit/powerdown failed - got timeout\n";
4817                 }
4818             } else {
4819                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4820                 return;
4821             }
4822         } else {
4823             if ($force) {
4824                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4825                 kill 15, $pid;
4826             } else {
4827                 die "VM quit/powerdown failed\n";
4828             }
4829         }
4830
4831         # wait again
4832         $timeout = 10;
4833
4834         my $count = 0;
4835         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4836             $count++;
4837             sleep 1;
4838         }
4839
4840         if ($count >= $timeout) {
4841             warn "VM still running - terminating now with SIGKILL\n";
4842             kill 9, $pid;
4843             sleep 1;
4844         }
4845
4846         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4847    });
4848 }
4849
4850 sub vm_suspend {
4851     my ($vmid, $skiplock) = @_;
4852
4853     PVE::QemuConfig->lock_config($vmid, sub {
4854
4855         my $conf = PVE::QemuConfig->load_config($vmid);
4856
4857         PVE::QemuConfig->check_lock($conf)
4858             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4859
4860         vm_mon_cmd($vmid, "stop");
4861     });
4862 }
4863
4864 sub vm_resume {
4865     my ($vmid, $skiplock, $nocheck) = @_;
4866
4867     PVE::QemuConfig->lock_config($vmid, sub {
4868
4869         if (!$nocheck) {
4870
4871             my $conf = PVE::QemuConfig->load_config($vmid);
4872
4873             PVE::QemuConfig->check_lock($conf)
4874                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4875
4876             vm_mon_cmd($vmid, "cont");
4877
4878         } else {
4879             vm_mon_cmd_nocheck($vmid, "cont");
4880         }
4881     });
4882 }
4883
4884 sub vm_sendkey {
4885     my ($vmid, $skiplock, $key) = @_;
4886
4887     PVE::QemuConfig->lock_config($vmid, sub {
4888
4889         my $conf = PVE::QemuConfig->load_config($vmid);
4890
4891         # there is no qmp command, so we use the human monitor command
4892         vm_human_monitor_command($vmid, "sendkey $key");
4893     });
4894 }
4895
4896 sub vm_destroy {
4897     my ($storecfg, $vmid, $skiplock) = @_;
4898
4899     PVE::QemuConfig->lock_config($vmid, sub {
4900
4901         my $conf = PVE::QemuConfig->load_config($vmid);
4902
4903         if (!check_running($vmid)) {
4904             destroy_vm($storecfg, $vmid, undef, $skiplock);
4905         } else {
4906             die "VM $vmid is running - destroy failed\n";
4907         }
4908     });
4909 }
4910
4911 # pci helpers
4912
4913 sub file_write {
4914     my ($filename, $buf) = @_;
4915
4916     my $fh = IO::File->new($filename, "w");
4917     return undef if !$fh;
4918
4919     my $res = print $fh $buf;
4920
4921     $fh->close();
4922
4923     return $res;
4924 }
4925
4926 sub pci_device_info {
4927     my ($name) = @_;
4928
4929     my $res;
4930
4931     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4932     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4933
4934     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4935     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4936
4937     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4938     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4939
4940     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4941     return undef if !defined($product) || $product !~ s/^0x//;
4942
4943     $res = {
4944         name => $name,
4945         vendor => $vendor,
4946         product => $product,
4947         domain => $domain,
4948         bus => $bus,
4949         slot => $slot,
4950         func => $func,
4951         irq => $irq,
4952         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4953     };
4954
4955     return $res;
4956 }
4957
4958 sub pci_dev_reset {
4959     my ($dev) = @_;
4960
4961     my $name = $dev->{name};
4962
4963     my $fn = "$pcisysfs/devices/$name/reset";
4964
4965     return file_write($fn, "1");
4966 }
4967
4968 sub pci_dev_bind_to_vfio {
4969     my ($dev) = @_;
4970
4971     my $name = $dev->{name};
4972
4973     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4974
4975     if (!-d $vfio_basedir) {
4976         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4977     }
4978     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4979
4980     my $testdir = "$vfio_basedir/$name";
4981     return 1 if -d $testdir;
4982
4983     my $data = "$dev->{vendor} $dev->{product}";
4984     return undef if !file_write("$vfio_basedir/new_id", $data);
4985
4986     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4987     if (!file_write($fn, $name)) {
4988         return undef if -f $fn;
4989     }
4990
4991     $fn = "$vfio_basedir/bind";
4992     if (! -d $testdir) {
4993         return undef if !file_write($fn, $name);
4994     }
4995
4996     return -d $testdir;
4997 }
4998
4999 sub pci_dev_group_bind_to_vfio {
5000     my ($pciid) = @_;
5001
5002     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5003
5004     if (!-d $vfio_basedir) {
5005         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5006     }
5007     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5008
5009     # get IOMMU group devices
5010     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5011       my @devs = grep /^0000:/, readdir($D);
5012     closedir($D);
5013
5014     foreach my $pciid (@devs) {
5015         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5016
5017         # pci bridges, switches or root ports are not supported
5018         # they have a pci_bus subdirectory so skip them
5019         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5020
5021         my $info = pci_device_info($1);
5022         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5023     }
5024
5025     return 1;
5026 }
5027
5028 sub print_pci_addr {
5029     my ($id, $bridges) = @_;
5030
5031     my $res = '';
5032     my $devices = {
5033         piix3 => { bus => 0, addr => 1 },
5034         #addr2 : first videocard
5035         balloon0 => { bus => 0, addr => 3 },
5036         watchdog => { bus => 0, addr => 4 },
5037         scsihw0 => { bus => 0, addr => 5 },
5038         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
5039         scsihw1 => { bus => 0, addr => 6 },
5040         ahci0 => { bus => 0, addr => 7 },
5041         qga0 => { bus => 0, addr => 8 },
5042         spice => { bus => 0, addr => 9 },
5043         virtio0 => { bus => 0, addr => 10 },
5044         virtio1 => { bus => 0, addr => 11 },
5045         virtio2 => { bus => 0, addr => 12 },
5046         virtio3 => { bus => 0, addr => 13 },
5047         virtio4 => { bus => 0, addr => 14 },
5048         virtio5 => { bus => 0, addr => 15 },
5049         hostpci0 => { bus => 0, addr => 16 },
5050         hostpci1 => { bus => 0, addr => 17 },
5051         net0 => { bus => 0, addr => 18 },
5052         net1 => { bus => 0, addr => 19 },
5053         net2 => { bus => 0, addr => 20 },
5054         net3 => { bus => 0, addr => 21 },
5055         net4 => { bus => 0, addr => 22 },
5056         net5 => { bus => 0, addr => 23 },
5057         vga1 => { bus => 0, addr => 24 },
5058         vga2 => { bus => 0, addr => 25 },
5059         vga3 => { bus => 0, addr => 26 },
5060         hostpci2 => { bus => 0, addr => 27 },
5061         hostpci3 => { bus => 0, addr => 28 },
5062         #addr29 : usb-host (pve-usb.cfg)
5063         'pci.1' => { bus => 0, addr => 30 },
5064         'pci.2' => { bus => 0, addr => 31 },
5065         'net6' => { bus => 1, addr => 1 },
5066         'net7' => { bus => 1, addr => 2 },
5067         'net8' => { bus => 1, addr => 3 },
5068         'net9' => { bus => 1, addr => 4 },
5069         'net10' => { bus => 1, addr => 5 },
5070         'net11' => { bus => 1, addr => 6 },
5071         'net12' => { bus => 1, addr => 7 },
5072         'net13' => { bus => 1, addr => 8 },
5073         'net14' => { bus => 1, addr => 9 },
5074         'net15' => { bus => 1, addr => 10 },
5075         'net16' => { bus => 1, addr => 11 },
5076         'net17' => { bus => 1, addr => 12 },
5077         'net18' => { bus => 1, addr => 13 },
5078         'net19' => { bus => 1, addr => 14 },
5079         'net20' => { bus => 1, addr => 15 },
5080         'net21' => { bus => 1, addr => 16 },
5081         'net22' => { bus => 1, addr => 17 },
5082         'net23' => { bus => 1, addr => 18 },
5083         'net24' => { bus => 1, addr => 19 },
5084         'net25' => { bus => 1, addr => 20 },
5085         'net26' => { bus => 1, addr => 21 },
5086         'net27' => { bus => 1, addr => 22 },
5087         'net28' => { bus => 1, addr => 23 },
5088         'net29' => { bus => 1, addr => 24 },
5089         'net30' => { bus => 1, addr => 25 },
5090         'net31' => { bus => 1, addr => 26 },
5091         'xhci' => { bus => 1, addr => 27 },
5092         'virtio6' => { bus => 2, addr => 1 },
5093         'virtio7' => { bus => 2, addr => 2 },
5094         'virtio8' => { bus => 2, addr => 3 },
5095         'virtio9' => { bus => 2, addr => 4 },
5096         'virtio10' => { bus => 2, addr => 5 },
5097         'virtio11' => { bus => 2, addr => 6 },
5098         'virtio12' => { bus => 2, addr => 7 },
5099         'virtio13' => { bus => 2, addr => 8 },
5100         'virtio14' => { bus => 2, addr => 9 },
5101         'virtio15' => { bus => 2, addr => 10 },
5102         'virtioscsi0' => { bus => 3, addr => 1 },
5103         'virtioscsi1' => { bus => 3, addr => 2 },
5104         'virtioscsi2' => { bus => 3, addr => 3 },
5105         'virtioscsi3' => { bus => 3, addr => 4 },
5106         'virtioscsi4' => { bus => 3, addr => 5 },
5107         'virtioscsi5' => { bus => 3, addr => 6 },
5108         'virtioscsi6' => { bus => 3, addr => 7 },
5109         'virtioscsi7' => { bus => 3, addr => 8 },
5110         'virtioscsi8' => { bus => 3, addr => 9 },
5111         'virtioscsi9' => { bus => 3, addr => 10 },
5112         'virtioscsi10' => { bus => 3, addr => 11 },
5113         'virtioscsi11' => { bus => 3, addr => 12 },
5114         'virtioscsi12' => { bus => 3, addr => 13 },
5115         'virtioscsi13' => { bus => 3, addr => 14 },
5116         'virtioscsi14' => { bus => 3, addr => 15 },
5117         'virtioscsi15' => { bus => 3, addr => 16 },
5118         'virtioscsi16' => { bus => 3, addr => 17 },
5119         'virtioscsi17' => { bus => 3, addr => 18 },
5120         'virtioscsi18' => { bus => 3, addr => 19 },
5121         'virtioscsi19' => { bus => 3, addr => 20 },
5122         'virtioscsi20' => { bus => 3, addr => 21 },
5123         'virtioscsi21' => { bus => 3, addr => 22 },
5124         'virtioscsi22' => { bus => 3, addr => 23 },
5125         'virtioscsi23' => { bus => 3, addr => 24 },
5126         'virtioscsi24' => { bus => 3, addr => 25 },
5127         'virtioscsi25' => { bus => 3, addr => 26 },
5128         'virtioscsi26' => { bus => 3, addr => 27 },
5129         'virtioscsi27' => { bus => 3, addr => 28 },
5130         'virtioscsi28' => { bus => 3, addr => 29 },
5131         'virtioscsi29' => { bus => 3, addr => 30 },
5132         'virtioscsi30' => { bus => 3, addr => 31 },
5133
5134     };
5135
5136     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5137            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5138            my $bus = $devices->{$id}->{bus};
5139            $res = ",bus=pci.$bus,addr=$addr";
5140            $bridges->{$bus} = 1 if $bridges;
5141     }
5142     return $res;
5143
5144 }
5145
5146 sub print_pcie_addr {
5147     my ($id) = @_;
5148
5149     my $res = '';
5150     my $devices = {
5151         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5152         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5153         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5154         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5155     };
5156
5157     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5158            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5159            my $bus = $devices->{$id}->{bus};
5160            $res = ",bus=$bus,addr=$addr";
5161     }
5162     return $res;
5163
5164 }
5165
5166 # vzdump restore implementaion
5167
5168 sub tar_archive_read_firstfile {
5169     my $archive = shift;
5170
5171     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5172
5173     # try to detect archive type first
5174     my $pid = open (TMP, "tar tf '$archive'|") ||
5175         die "unable to open file '$archive'\n";
5176     my $firstfile = <TMP>;
5177     kill 15, $pid;
5178     close TMP;
5179
5180     die "ERROR: archive contaions no data\n" if !$firstfile;
5181     chomp $firstfile;
5182
5183     return $firstfile;
5184 }
5185
5186 sub tar_restore_cleanup {
5187     my ($storecfg, $statfile) = @_;
5188
5189     print STDERR "starting cleanup\n";
5190
5191     if (my $fd = IO::File->new($statfile, "r")) {
5192         while (defined(my $line = <$fd>)) {
5193             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5194                 my $volid = $2;
5195                 eval {
5196                     if ($volid =~ m|^/|) {
5197                         unlink $volid || die 'unlink failed\n';
5198                     } else {
5199                         PVE::Storage::vdisk_free($storecfg, $volid);
5200                     }
5201                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5202                 };
5203                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5204             } else {
5205                 print STDERR "unable to parse line in statfile - $line";
5206             }
5207         }
5208         $fd->close();
5209     }
5210 }
5211
5212 sub restore_archive {
5213     my ($archive, $vmid, $user, $opts) = @_;
5214
5215     my $format = $opts->{format};
5216     my $comp;
5217
5218     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5219         $format = 'tar' if !$format;
5220         $comp = 'gzip';
5221     } elsif ($archive =~ m/\.tar$/) {
5222         $format = 'tar' if !$format;
5223     } elsif ($archive =~ m/.tar.lzo$/) {
5224         $format = 'tar' if !$format;
5225         $comp = 'lzop';
5226     } elsif ($archive =~ m/\.vma$/) {
5227         $format = 'vma' if !$format;
5228     } elsif ($archive =~ m/\.vma\.gz$/) {
5229         $format = 'vma' if !$format;
5230         $comp = 'gzip';
5231     } elsif ($archive =~ m/\.vma\.lzo$/) {
5232         $format = 'vma' if !$format;
5233         $comp = 'lzop';
5234     } else {
5235         $format = 'vma' if !$format; # default
5236     }
5237
5238     # try to detect archive format
5239     if ($format eq 'tar') {
5240         return restore_tar_archive($archive, $vmid, $user, $opts);
5241     } else {
5242         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5243     }
5244 }
5245
5246 sub restore_update_config_line {
5247     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5248
5249     return if $line =~ m/^\#qmdump\#/;
5250     return if $line =~ m/^\#vzdump\#/;
5251     return if $line =~ m/^lock:/;
5252     return if $line =~ m/^unused\d+:/;
5253     return if $line =~ m/^parent:/;
5254     return if $line =~ m/^template:/; # restored VM is never a template
5255
5256     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5257         # try to convert old 1.X settings
5258         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5259         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5260             my ($model, $macaddr) = split(/\=/, $devconfig);
5261             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5262             my $net = {
5263                 model => $model,
5264                 bridge => "vmbr$ind",
5265                 macaddr => $macaddr,
5266             };
5267             my $netstr = print_net($net);
5268
5269             print $outfd "net$cookie->{netcount}: $netstr\n";
5270             $cookie->{netcount}++;
5271         }
5272     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5273         my ($id, $netstr) = ($1, $2);
5274         my $net = parse_net($netstr);
5275         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5276         $netstr = print_net($net);
5277         print $outfd "$id: $netstr\n";
5278     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5279         my $virtdev = $1;
5280         my $value = $3;
5281         my $di = parse_drive($virtdev, $value);
5282         if (defined($di->{backup}) && !$di->{backup}) {
5283             print $outfd "#$line";
5284         } elsif ($map->{$virtdev}) {
5285             delete $di->{format}; # format can change on restore
5286             $di->{file} = $map->{$virtdev};
5287             $value = print_drive($vmid, $di);
5288             print $outfd "$virtdev: $value\n";
5289         } else {
5290             print $outfd $line;
5291         }
5292     } else {
5293         print $outfd $line;
5294     }
5295 }
5296
5297 sub scan_volids {
5298     my ($cfg, $vmid) = @_;
5299
5300     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5301
5302     my $volid_hash = {};
5303     foreach my $storeid (keys %$info) {
5304         foreach my $item (@{$info->{$storeid}}) {
5305             next if !($item->{volid} && $item->{size});
5306             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5307             $volid_hash->{$item->{volid}} = $item;
5308         }
5309     }
5310
5311     return $volid_hash;
5312 }
5313
5314 sub is_volume_in_use {
5315     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5316
5317     my $path = PVE::Storage::path($storecfg, $volid);
5318
5319     my $scan_config = sub {
5320         my ($cref, $snapname) = @_;
5321
5322         foreach my $key (keys %$cref) {
5323             my $value = $cref->{$key};
5324             if (is_valid_drivename($key)) {
5325                 next if $skip_drive && $key eq $skip_drive;
5326                 my $drive = parse_drive($key, $value);
5327                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5328                 return 1 if $volid eq $drive->{file};
5329                 if ($drive->{file} =~ m!^/!) {
5330                     return 1 if $drive->{file} eq $path;
5331                 } else {
5332                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5333                     next if !$storeid;
5334                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5335                     next if !$scfg;
5336                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5337                 }
5338             }
5339         }
5340
5341         return 0;
5342     };
5343
5344     return 1 if &$scan_config($conf);
5345
5346     undef $skip_drive;
5347
5348     foreach my $snapname (keys %{$conf->{snapshots}}) {
5349         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5350     }
5351
5352     return 0;
5353 }
5354
5355 sub update_disksize {
5356     my ($vmid, $conf, $volid_hash) = @_;
5357
5358     my $changes;
5359
5360     my $used = {};
5361
5362     # Note: it is allowed to define multiple storages with same path (alias), so
5363     # we need to check both 'volid' and real 'path' (two different volid can point
5364     # to the same path).
5365
5366     my $usedpath = {};
5367
5368     # update size info
5369     foreach my $opt (keys %$conf) {
5370         if (is_valid_drivename($opt)) {
5371             my $drive = parse_drive($opt, $conf->{$opt});
5372             my $volid = $drive->{file};
5373             next if !$volid;
5374
5375             $used->{$volid} = 1;
5376             if ($volid_hash->{$volid} &&
5377                 (my $path = $volid_hash->{$volid}->{path})) {
5378                 $usedpath->{$path} = 1;
5379             }
5380
5381             next if drive_is_cdrom($drive);
5382             next if !$volid_hash->{$volid};
5383
5384             $drive->{size} = $volid_hash->{$volid}->{size};
5385             my $new = print_drive($vmid, $drive);
5386             if ($new ne $conf->{$opt}) {
5387                 $changes = 1;
5388                 $conf->{$opt} = $new;
5389             }
5390         }
5391     }
5392
5393     # remove 'unusedX' entry if volume is used
5394     foreach my $opt (keys %$conf) {
5395         next if $opt !~ m/^unused\d+$/;
5396         my $volid = $conf->{$opt};
5397         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5398         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5399             $changes = 1;
5400             delete $conf->{$opt};
5401         }
5402     }
5403
5404     foreach my $volid (sort keys %$volid_hash) {
5405         next if $volid =~ m/vm-$vmid-state-/;
5406         next if $used->{$volid};
5407         my $path = $volid_hash->{$volid}->{path};
5408         next if !$path; # just to be sure
5409         next if $usedpath->{$path};
5410         $changes = 1;
5411         PVE::QemuConfig->add_unused_volume($conf, $volid);
5412         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5413     }
5414
5415     return $changes;
5416 }
5417
5418 sub rescan {
5419     my ($vmid, $nolock) = @_;
5420
5421     my $cfg = PVE::Storage::config();
5422
5423     my $volid_hash = scan_volids($cfg, $vmid);
5424
5425     my $updatefn =  sub {
5426         my ($vmid) = @_;
5427
5428         my $conf = PVE::QemuConfig->load_config($vmid);
5429
5430         PVE::QemuConfig->check_lock($conf);
5431
5432         my $vm_volids = {};
5433         foreach my $volid (keys %$volid_hash) {
5434             my $info = $volid_hash->{$volid};
5435             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5436         }
5437
5438         my $changes = update_disksize($vmid, $conf, $vm_volids);
5439
5440         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5441     };
5442
5443     if (defined($vmid)) {
5444         if ($nolock) {
5445             &$updatefn($vmid);
5446         } else {
5447             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5448         }
5449     } else {
5450         my $vmlist = config_list();
5451         foreach my $vmid (keys %$vmlist) {
5452             if ($nolock) {
5453                 &$updatefn($vmid);
5454             } else {
5455                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5456             }
5457         }
5458     }
5459 }
5460
5461 sub restore_vma_archive {
5462     my ($archive, $vmid, $user, $opts, $comp) = @_;
5463
5464     my $input = $archive eq '-' ? "<&STDIN" : undef;
5465     my $readfrom = $archive;
5466
5467     my $uncomp = '';
5468     if ($comp) {
5469         $readfrom = '-';
5470         my $qarchive = PVE::Tools::shellquote($archive);
5471         if ($comp eq 'gzip') {
5472             $uncomp = "zcat $qarchive|";
5473         } elsif ($comp eq 'lzop') {
5474             $uncomp = "lzop -d -c $qarchive|";
5475         } else {
5476             die "unknown compression method '$comp'\n";
5477         }
5478
5479     }
5480
5481     my $tmpdir = "/var/tmp/vzdumptmp$$";
5482     rmtree $tmpdir;
5483
5484     # disable interrupts (always do cleanups)
5485     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5486         warn "got interrupt - ignored\n";
5487     };
5488
5489     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5490     POSIX::mkfifo($mapfifo, 0600);
5491     my $fifofh;
5492
5493     my $openfifo = sub {
5494         open($fifofh, '>', $mapfifo) || die $!;
5495     };
5496
5497     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5498
5499     my $oldtimeout;
5500     my $timeout = 5;
5501
5502     my $devinfo = {};
5503
5504     my $rpcenv = PVE::RPCEnvironment::get();
5505
5506     my $conffile = PVE::QemuConfig->config_file($vmid);
5507     my $tmpfn = "$conffile.$$.tmp";
5508
5509     # Note: $oldconf is undef if VM does not exists
5510     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5511     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5512
5513     my $print_devmap = sub {
5514         my $virtdev_hash = {};
5515
5516         my $cfgfn = "$tmpdir/qemu-server.conf";
5517
5518         # we can read the config - that is already extracted
5519         my $fh = IO::File->new($cfgfn, "r") ||
5520             "unable to read qemu-server.conf - $!\n";
5521
5522         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5523         if (-f $fwcfgfn) {
5524             my $pve_firewall_dir = '/etc/pve/firewall';
5525             mkdir $pve_firewall_dir; # make sure the dir exists
5526             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5527         }
5528
5529         while (defined(my $line = <$fh>)) {
5530             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5531                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5532                 die "archive does not contain data for drive '$virtdev'\n"
5533                     if !$devinfo->{$devname};
5534                 if (defined($opts->{storage})) {
5535                     $storeid = $opts->{storage} || 'local';
5536                 } elsif (!$storeid) {
5537                     $storeid = 'local';
5538                 }
5539                 $format = 'raw' if !$format;
5540                 $devinfo->{$devname}->{devname} = $devname;
5541                 $devinfo->{$devname}->{virtdev} = $virtdev;
5542                 $devinfo->{$devname}->{format} = $format;
5543                 $devinfo->{$devname}->{storeid} = $storeid;
5544
5545                 # check permission on storage
5546                 my $pool = $opts->{pool}; # todo: do we need that?
5547                 if ($user ne 'root@pam') {
5548                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5549                 }
5550
5551                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5552             }
5553         }
5554
5555         foreach my $devname (keys %$devinfo) {
5556             die "found no device mapping information for device '$devname'\n"
5557                 if !$devinfo->{$devname}->{virtdev};
5558         }
5559
5560         my $cfg = PVE::Storage::config();
5561
5562         # create empty/temp config
5563         if ($oldconf) {
5564             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5565             foreach_drive($oldconf, sub {
5566                 my ($ds, $drive) = @_;
5567
5568                 return if drive_is_cdrom($drive);
5569
5570                 my $volid = $drive->{file};
5571
5572                 return if !$volid || $volid =~ m|^/|;
5573
5574                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5575                 return if !$path || !$owner || ($owner != $vmid);
5576
5577                 # Note: only delete disk we want to restore
5578                 # other volumes will become unused
5579                 if ($virtdev_hash->{$ds}) {
5580                     PVE::Storage::vdisk_free($cfg, $volid);
5581                 }
5582             });
5583
5584             # delete vmstate files
5585             # since after the restore we have no snapshots anymore
5586             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5587                 my $snap = $oldconf->{snapshots}->{$snapname};
5588                 if ($snap->{vmstate}) {
5589                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5590                     if (my $err = $@) {
5591                         warn $err;
5592                     }
5593                 }
5594             }
5595         }
5596
5597         my $map = {};
5598         foreach my $virtdev (sort keys %$virtdev_hash) {
5599             my $d = $virtdev_hash->{$virtdev};
5600             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5601             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5602
5603             # test if requested format is supported
5604             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5605             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5606             $d->{format} = $defFormat if !$supported;
5607
5608             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5609                                                   $d->{format}, undef, $alloc_size);
5610             print STDERR "new volume ID is '$volid'\n";
5611             $d->{volid} = $volid;
5612             my $path = PVE::Storage::path($cfg, $volid);
5613
5614             PVE::Storage::activate_volumes($cfg,[$volid]);
5615
5616             my $write_zeros = 1;
5617             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5618                 $write_zeros = 0;
5619             }
5620
5621             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5622
5623             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5624             $map->{$virtdev} = $volid;
5625         }
5626
5627         $fh->seek(0, 0) || die "seek failed - $!\n";
5628
5629         my $outfd = new IO::File ($tmpfn, "w") ||
5630             die "unable to write config for VM $vmid\n";
5631
5632         my $cookie = { netcount => 0 };
5633         while (defined(my $line = <$fh>)) {
5634             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5635         }
5636
5637         $fh->close();
5638         $outfd->close();
5639     };
5640
5641     eval {
5642         # enable interrupts
5643         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5644             die "interrupted by signal\n";
5645         };
5646         local $SIG{ALRM} = sub { die "got timeout\n"; };
5647
5648         $oldtimeout = alarm($timeout);
5649
5650         my $parser = sub {
5651             my $line = shift;
5652
5653             print "$line\n";
5654
5655             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5656                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5657                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5658             } elsif ($line =~ m/^CTIME: /) {
5659                 # we correctly received the vma config, so we can disable
5660                 # the timeout now for disk allocation (set to 10 minutes, so
5661                 # that we always timeout if something goes wrong)
5662                 alarm(600);
5663                 &$print_devmap();
5664                 print $fifofh "done\n";
5665                 my $tmp = $oldtimeout || 0;
5666                 $oldtimeout = undef;
5667                 alarm($tmp);
5668                 close($fifofh);
5669             }
5670         };
5671
5672         print "restore vma archive: $cmd\n";
5673         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5674     };
5675     my $err = $@;
5676
5677     alarm($oldtimeout) if $oldtimeout;
5678
5679     my $vollist = [];
5680     foreach my $devname (keys %$devinfo) {
5681         my $volid = $devinfo->{$devname}->{volid};
5682         push @$vollist, $volid if $volid;
5683     }
5684
5685     my $cfg = PVE::Storage::config();
5686     PVE::Storage::deactivate_volumes($cfg, $vollist);
5687
5688     unlink $mapfifo;
5689
5690     if ($err) {
5691         rmtree $tmpdir;
5692         unlink $tmpfn;
5693
5694         foreach my $devname (keys %$devinfo) {
5695             my $volid = $devinfo->{$devname}->{volid};
5696             next if !$volid;
5697             eval {
5698                 if ($volid =~ m|^/|) {
5699                     unlink $volid || die 'unlink failed\n';
5700                 } else {
5701                     PVE::Storage::vdisk_free($cfg, $volid);
5702                 }
5703                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5704             };
5705             print STDERR "unable to cleanup '$volid' - $@" if $@;
5706         }
5707         die $err;
5708     }
5709
5710     rmtree $tmpdir;
5711
5712     rename($tmpfn, $conffile) ||
5713         die "unable to commit configuration file '$conffile'\n";
5714
5715     PVE::Cluster::cfs_update(); # make sure we read new file
5716
5717     eval { rescan($vmid, 1); };
5718     warn $@ if $@;
5719 }
5720
5721 sub restore_tar_archive {
5722     my ($archive, $vmid, $user, $opts) = @_;
5723
5724     if ($archive ne '-') {
5725         my $firstfile = tar_archive_read_firstfile($archive);
5726         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5727             if $firstfile ne 'qemu-server.conf';
5728     }
5729
5730     my $storecfg = PVE::Storage::config();
5731
5732     # destroy existing data - keep empty config
5733     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5734     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5735
5736     my $tocmd = "/usr/lib/qemu-server/qmextract";
5737
5738     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5739     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5740     $tocmd .= ' --prealloc' if $opts->{prealloc};
5741     $tocmd .= ' --info' if $opts->{info};
5742
5743     # tar option "xf" does not autodetect compression when read from STDIN,
5744     # so we pipe to zcat
5745     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5746         PVE::Tools::shellquote("--to-command=$tocmd");
5747
5748     my $tmpdir = "/var/tmp/vzdumptmp$$";
5749     mkpath $tmpdir;
5750
5751     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5752     local $ENV{VZDUMP_VMID} = $vmid;
5753     local $ENV{VZDUMP_USER} = $user;
5754
5755     my $conffile = PVE::QemuConfig->config_file($vmid);
5756     my $tmpfn = "$conffile.$$.tmp";
5757
5758     # disable interrupts (always do cleanups)
5759     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5760         print STDERR "got interrupt - ignored\n";
5761     };
5762
5763     eval {
5764         # enable interrupts
5765         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5766             die "interrupted by signal\n";
5767         };
5768
5769         if ($archive eq '-') {
5770             print "extracting archive from STDIN\n";
5771             run_command($cmd, input => "<&STDIN");
5772         } else {
5773             print "extracting archive '$archive'\n";
5774             run_command($cmd);
5775         }
5776
5777         return if $opts->{info};
5778
5779         # read new mapping
5780         my $map = {};
5781         my $statfile = "$tmpdir/qmrestore.stat";
5782         if (my $fd = IO::File->new($statfile, "r")) {
5783             while (defined (my $line = <$fd>)) {
5784                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5785                     $map->{$1} = $2 if $1;
5786                 } else {
5787                     print STDERR "unable to parse line in statfile - $line\n";
5788                 }
5789             }
5790             $fd->close();
5791         }
5792
5793         my $confsrc = "$tmpdir/qemu-server.conf";
5794
5795         my $srcfd = new IO::File($confsrc, "r") ||
5796             die "unable to open file '$confsrc'\n";
5797
5798         my $outfd = new IO::File ($tmpfn, "w") ||
5799             die "unable to write config for VM $vmid\n";
5800
5801         my $cookie = { netcount => 0 };
5802         while (defined (my $line = <$srcfd>)) {
5803             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5804         }
5805
5806         $srcfd->close();
5807         $outfd->close();
5808     };
5809     my $err = $@;
5810
5811     if ($err) {
5812
5813         unlink $tmpfn;
5814
5815         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5816
5817         die $err;
5818     }
5819
5820     rmtree $tmpdir;
5821
5822     rename $tmpfn, $conffile ||
5823         die "unable to commit configuration file '$conffile'\n";
5824
5825     PVE::Cluster::cfs_update(); # make sure we read new file
5826
5827     eval { rescan($vmid, 1); };
5828     warn $@ if $@;
5829 };
5830
5831 sub foreach_writable_storage {
5832     my ($conf, $func) = @_;
5833
5834     my $sidhash = {};
5835
5836     foreach my $ds (keys %$conf) {
5837         next if !is_valid_drivename($ds);
5838
5839         my $drive = parse_drive($ds, $conf->{$ds});
5840         next if !$drive;
5841         next if drive_is_cdrom($drive);
5842
5843         my $volid = $drive->{file};
5844
5845         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5846         $sidhash->{$sid} = $sid if $sid;
5847     }
5848
5849     foreach my $sid (sort keys %$sidhash) {
5850         &$func($sid);
5851     }
5852 }
5853
5854 sub do_snapshots_with_qemu {
5855     my ($storecfg, $volid) = @_;
5856
5857     my $storage_name = PVE::Storage::parse_volume_id($volid);
5858
5859     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5860         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5861         return 1;
5862     }
5863
5864     if ($volid =~ m/\.(qcow2|qed)$/){
5865         return 1;
5866     }
5867
5868     return undef;
5869 }
5870
5871 sub qga_check_running {
5872     my ($vmid) = @_;
5873
5874     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5875     if ($@) {
5876         warn "Qemu Guest Agent are not running - $@";
5877         return 0;
5878     }
5879     return 1;
5880 }
5881
5882 sub template_create {
5883     my ($vmid, $conf, $disk) = @_;
5884
5885     my $storecfg = PVE::Storage::config();
5886
5887     foreach_drive($conf, sub {
5888         my ($ds, $drive) = @_;
5889
5890         return if drive_is_cdrom($drive);
5891         return if $disk && $ds ne $disk;
5892
5893         my $volid = $drive->{file};
5894         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5895
5896         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5897         $drive->{file} = $voliddst;
5898         $conf->{$ds} = print_drive($vmid, $drive);
5899         PVE::QemuConfig->write_config($vmid, $conf);
5900     });
5901 }
5902
5903 sub qemu_img_convert {
5904     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5905
5906     my $storecfg = PVE::Storage::config();
5907     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5908     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5909
5910     if ($src_storeid && $dst_storeid) {
5911
5912         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5913
5914         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5915         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5916
5917         my $src_format = qemu_img_format($src_scfg, $src_volname);
5918         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5919
5920         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5921         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5922
5923         my $cmd = [];
5924         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5925         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5926         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5927         if ($is_zero_initialized) {
5928             push @$cmd, "zeroinit:$dst_path";
5929         } else {
5930             push @$cmd, $dst_path;
5931         }
5932
5933         my $parser = sub {
5934             my $line = shift;
5935             if($line =~ m/\((\S+)\/100\%\)/){
5936                 my $percent = $1;
5937                 my $transferred = int($size * $percent / 100);
5938                 my $remaining = $size - $transferred;
5939
5940                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5941             }
5942
5943         };
5944
5945         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5946         my $err = $@;
5947         die "copy failed: $err" if $err;
5948     }
5949 }
5950
5951 sub qemu_img_format {
5952     my ($scfg, $volname) = @_;
5953
5954     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5955         return $1;
5956     } else {
5957         return "raw";
5958     }
5959 }
5960
5961 sub qemu_drive_mirror {
5962     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5963
5964     my $storecfg = PVE::Storage::config();
5965     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5966
5967     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5968
5969     my $format = qemu_img_format($dst_scfg, $dst_volname);
5970
5971     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5972
5973     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5974
5975     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5976     $opts->{format} = $format if $format;
5977
5978     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5979
5980     my $finish_job = sub {
5981         while (1) {
5982             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5983             my $stat = @$stats[0];
5984             last if !$stat;
5985             sleep 1;
5986         }
5987     };
5988
5989     eval {
5990     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5991         while (1) {
5992             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5993             my $stat = @$stats[0];
5994             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5995             die "error job is not mirroring" if $stat->{type} ne "mirror";
5996
5997             my $busy = $stat->{busy};
5998             my $ready = $stat->{ready};
5999
6000             if (my $total = $stat->{len}) {
6001                 my $transferred = $stat->{offset} || 0;
6002                 my $remaining = $total - $transferred;
6003                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6004
6005                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6006             }
6007
6008
6009             if ($stat->{ready} eq 'true') {
6010
6011                 last if $vmiddst != $vmid;
6012
6013                 # try to switch the disk if source and destination are on the same guest
6014                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6015                 if (!$@) {
6016                     &$finish_job();
6017                     last;
6018                 }
6019                 die $@ if $@ !~ m/cannot be completed/;
6020             }
6021             sleep 1;
6022         }
6023
6024
6025     };
6026     my $err = $@;
6027
6028     my $cancel_job = sub {
6029         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6030         &$finish_job();
6031     };
6032
6033     if ($err) {
6034         eval { &$cancel_job(); };
6035         die "mirroring error: $err";
6036     }
6037
6038     if ($vmiddst != $vmid) {
6039         # if we clone a disk for a new target vm, we don't switch the disk
6040         &$cancel_job(); # so we call block-job-cancel
6041     }
6042 }
6043
6044 sub clone_disk {
6045     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6046         $newvmid, $storage, $format, $full, $newvollist) = @_;
6047
6048     my $newvolid;
6049
6050     if (!$full) {
6051         print "create linked clone of drive $drivename ($drive->{file})\n";
6052         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6053         push @$newvollist, $newvolid;
6054     } else {
6055         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6056         $storeid = $storage if $storage;
6057
6058         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6059         if (!$format) {
6060             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6061             $format = qemu_img_format($scfg, $volname);
6062         }
6063
6064         # test if requested format is supported - else use default
6065         my $supported = grep { $_ eq $format } @$validFormats;
6066         $format = $defFormat if !$supported;
6067
6068         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6069
6070         print "create full clone of drive $drivename ($drive->{file})\n";
6071         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6072         push @$newvollist, $newvolid;
6073
6074         PVE::Storage::activate_volumes($storecfg, $newvollist);
6075
6076         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6077         if (!$running || $snapname) {
6078             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6079         } else {
6080             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
6081         }
6082     }
6083
6084     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6085
6086     my $disk = $drive;
6087     $disk->{format} = undef;
6088     $disk->{file} = $newvolid;
6089     $disk->{size} = $size;
6090
6091     return $disk;
6092 }
6093
6094 # this only works if VM is running
6095 sub get_current_qemu_machine {
6096     my ($vmid) = @_;
6097
6098     my $cmd = { execute => 'query-machines', arguments => {} };
6099     my $res = vm_qmp_command($vmid, $cmd);
6100
6101     my ($current, $default);
6102     foreach my $e (@$res) {
6103         $default = $e->{name} if $e->{'is-default'};
6104         $current = $e->{name} if $e->{'is-current'};
6105     }
6106
6107     # fallback to the default machine if current is not supported by qemu
6108     return $current || $default || 'pc';
6109 }
6110
6111 sub qemu_machine_feature_enabled {
6112     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6113
6114     my $current_major;
6115     my $current_minor;
6116
6117     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6118
6119         $current_major = $3;
6120         $current_minor = $4;
6121
6122     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6123
6124         $current_major = $1;
6125         $current_minor = $2;
6126     }
6127
6128     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6129
6130
6131 }
6132
6133 sub qemu_machine_pxe {
6134     my ($vmid, $conf, $machine) = @_;
6135
6136     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6137
6138     foreach my $opt (keys %$conf) {
6139         next if $opt !~ m/^net(\d+)$/;
6140         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6141         next if !$net;
6142         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6143         return $machine.".pxe" if $romfile =~ m/pxe/;
6144         last;
6145     }
6146
6147     return $machine;
6148 }
6149
6150 sub qemu_use_old_bios_files {
6151     my ($machine_type) = @_;
6152
6153     return if !$machine_type;
6154
6155     my $use_old_bios_files = undef;
6156
6157     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6158         $machine_type = $1;
6159         $use_old_bios_files = 1;
6160     } else {
6161         my $kvmver = kvm_user_version();
6162         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6163         # load new efi bios files on migration. So this hack is required to allow
6164         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6165         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6166         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6167     }
6168
6169     return ($use_old_bios_files, $machine_type);
6170 }
6171
6172 sub lspci {
6173
6174     my $devices = {};
6175
6176     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6177             my (undef, $id, $function) = @_;
6178             my $res = { id => $id, function => $function};
6179             push @{$devices->{$id}}, $res;
6180     });
6181
6182     return $devices;
6183 }
6184
6185 sub vm_iothreads_list {
6186     my ($vmid) = @_;
6187
6188     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6189
6190     my $iothreads = {};
6191     foreach my $iothread (@$res) {
6192         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6193     }
6194
6195     return $iothreads;
6196 }
6197
6198 sub scsihw_infos {
6199     my ($conf, $drive) = @_;
6200
6201     my $maxdev = 0;
6202
6203     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6204         $maxdev = 7;
6205     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6206         $maxdev = 1;
6207     } else {
6208         $maxdev = 256;
6209     }
6210
6211     my $controller = int($drive->{index} / $maxdev);
6212     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6213
6214     return ($maxdev, $controller, $controller_prefix);
6215 }
6216
6217 # bash completion helper
6218
6219 sub complete_backup_archives {
6220     my ($cmdname, $pname, $cvalue) = @_;
6221
6222     my $cfg = PVE::Storage::config();
6223
6224     my $storeid;
6225
6226     if ($cvalue =~ m/^([^:]+):/) {
6227         $storeid = $1;
6228     }
6229
6230     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6231
6232     my $res = [];
6233     foreach my $id (keys %$data) {
6234         foreach my $item (@{$data->{$id}}) {
6235             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6236             push @$res, $item->{volid} if defined($item->{volid});
6237         }
6238     }
6239
6240     return $res;
6241 }
6242
6243 my $complete_vmid_full = sub {
6244     my ($running) = @_;
6245
6246     my $idlist = vmstatus();
6247
6248     my $res = [];
6249
6250     foreach my $id (keys %$idlist) {
6251         my $d = $idlist->{$id};
6252         if (defined($running)) {
6253             next if $d->{template};
6254             next if $running && $d->{status} ne 'running';
6255             next if !$running && $d->{status} eq 'running';
6256         }
6257         push @$res, $id;
6258
6259     }
6260     return $res;
6261 };
6262
6263 sub complete_vmid {
6264     return &$complete_vmid_full();
6265 }
6266
6267 sub complete_vmid_stopped {
6268     return &$complete_vmid_full(0);
6269 }
6270
6271 sub complete_vmid_running {
6272     return &$complete_vmid_full(1);
6273 }
6274
6275 sub complete_storage {
6276
6277     my $cfg = PVE::Storage::config();
6278     my $ids = $cfg->{ids};
6279
6280     my $res = [];
6281     foreach my $sid (keys %$ids) {
6282         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6283         next if !$ids->{$sid}->{content}->{images};
6284         push @$res, $sid;
6285     }
6286
6287     return $res;
6288 }
6289
6290 1;