PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  41
  42 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  43
  44 # Note about locking: we use flock on the config file protect
  45 # against concurent actions.
  46 # Aditionaly, we have a 'lock' setting in the config file. This
  47 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  48 # allowed when such lock is set. But you can ignore this kind of
  49 # lock with the --skiplock flag.
  50
  51 cfs_register_file('/qemu-server/',
  52                   \&parse_vm_config,
  53                   \&write_vm_config);
  54
  55 PVE::JSONSchema::register_standard_option('skiplock', {
  56     description => "Ignore locks - only root is allowed to use this option.",
  57     type => 'boolean',
  58     optional => 1,
  59 });
  60
  61 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  62     description => "Some command save/restore state from this location.",
  63     type => 'string',
  64     maxLength => 128,
  65     optional => 1,
  66 });
  67
  68 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  69     description => "The name of the snapshot.",
  70     type => 'string', format => 'pve-configid',
  71     maxLength => 40,
  72 });
  73
  74 #no warnings 'redefine';
  75
  76 sub cgroups_write {
  77    my ($controller, $vmid, $option, $value) = @_;
  78
  79    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  80    PVE::ProcFSTools::write_proc_entry($path, $value);
  81
  82 }
  83
  84 my $nodename = PVE::INotify::nodename();
  85
  86 mkdir "/etc/pve/nodes/$nodename";
  87 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  88 mkdir $confdir;
  89
  90 my $var_run_tmpdir = "/var/run/qemu-server";
  91 mkdir $var_run_tmpdir;
  92
  93 my $lock_dir = "/var/lock/qemu-server";
  94 mkdir $lock_dir;
  95
  96 my $pcisysfs = "/sys/bus/pci";
  97
  98 my $cpu_vendor_list = {
  99     # Intel CPUs
 100     486 => 'GenuineIntel',
 101     pentium => 'GenuineIntel',
 102     pentium2  => 'GenuineIntel',
 103     pentium3  => 'GenuineIntel',
 104     coreduo => 'GenuineIntel',
 105     core2duo => 'GenuineIntel',
 106     Conroe  => 'GenuineIntel',
 107     Penryn  => 'GenuineIntel',
 108     Nehalem  => 'GenuineIntel',
 109     Westmere => 'GenuineIntel',
 110     SandyBridge => 'GenuineIntel',
 111     IvyBridge => 'GenuineIntel',
 112     Haswell => 'GenuineIntel',
 113     'Haswell-noTSX' => 'GenuineIntel',
 114     Broadwell => 'GenuineIntel',
 115     'Broadwell-noTSX' => 'GenuineIntel',
 116
 117     # AMD CPUs
 118     athlon => 'AuthenticAMD',
 119     phenom  => 'AuthenticAMD',
 120     Opteron_G1  => 'AuthenticAMD',
 121     Opteron_G2  => 'AuthenticAMD',
 122     Opteron_G3  => 'AuthenticAMD',
 123     Opteron_G4  => 'AuthenticAMD',
 124     Opteron_G5  => 'AuthenticAMD',
 125
 126     # generic types, use vendor from host node
 127     host => 'default',
 128     kvm32 => 'default',
 129     kvm64 => 'default',
 130     qemu32 => 'default',
 131     qemu64 => 'default',
 132 };
 133
 134 my $cpu_fmt = {
 135     cputype => {
 136         description => "Emulated CPU type.",
 137         type => 'string',
 138         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 139         format_description => 'cputype',
 140         default => 'kvm64',
 141         default_key => 1,
 142     },
 143     hidden => {
 144         description => "Do not identify as a KVM virtual machine.",
 145         type => 'boolean',
 146         optional => 1,
 147         default => 0
 148     },
 149 };
 150
 151 my $watchdog_fmt = {
 152     model => {
 153         default_key => 1,
 154         type => 'string',
 155         enum => [qw(i6300esb ib700)],
 156         description => "Watchdog type to emulate.",
 157         default => 'i6300esb',
 158         optional => 1,
 159     },
 160     action => {
 161         type => 'string',
 162         enum => [qw(reset shutdown poweroff pause debug none)],
 163         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 164         optional => 1,
 165     },
 166 };
 167 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 168
 169 my $confdesc = {
 170     onboot => {
 171         optional => 1,
 172         type => 'boolean',
 173         description => "Specifies whether a VM will be started during system bootup.",
 174         default => 0,
 175     },
 176     autostart => {
 177         optional => 1,
 178         type => 'boolean',
 179         description => "Automatic restart after crash (currently ignored).",
 180         default => 0,
 181     },
 182     hotplug => {
 183         optional => 1,
 184         type => 'string', format => 'pve-hotplug-features',
 185         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 186         default => 'network,disk,usb',
 187     },
 188     reboot => {
 189         optional => 1,
 190         type => 'boolean',
 191         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 192         default => 1,
 193     },
 194     lock => {
 195         optional => 1,
 196         type => 'string',
 197         description => "Lock/unlock the VM.",
 198         enum => [qw(migrate backup snapshot rollback)],
 199     },
 200     cpulimit => {
 201         optional => 1,
 202         type => 'number',
 203         description => "Limit of CPU usage.",
 204         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 205         minimum => 0,
 206         maximum => 128,
 207         default => 0,
 208     },
 209     cpuunits => {
 210         optional => 1,
 211         type => 'integer',
 212         description => "CPU weight for a VM.",
 213         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 214         minimum => 0,
 215         maximum => 500000,
 216         default => 1000,
 217     },
 218     memory => {
 219         optional => 1,
 220         type => 'integer',
 221         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 222         minimum => 16,
 223         default => 512,
 224     },
 225     balloon => {
 226         optional => 1,
 227         type => 'integer',
 228         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 229         minimum => 0,
 230     },
 231     shares => {
 232         optional => 1,
 233         type => 'integer',
 234         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 235         minimum => 0,
 236         maximum => 50000,
 237         default => 1000,
 238     },
 239     keyboard => {
 240         optional => 1,
 241         type => 'string',
 242         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 243         enum => PVE::Tools::kvmkeymaplist(),
 244         default => 'en-us',
 245     },
 246     name => {
 247         optional => 1,
 248         type => 'string', format => 'dns-name',
 249         description => "Set a name for the VM. Only used on the configuration web interface.",
 250     },
 251     scsihw => {
 252         optional => 1,
 253         type => 'string',
 254         description => "SCSI controller model",
 255         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 256         default => 'lsi',
 257     },
 258     description => {
 259         optional => 1,
 260         type => 'string',
 261         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 262     },
 263     ostype => {
 264         optional => 1,
 265         type => 'string',
 266         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 267         description => "Specify guest operating system.",
 268         verbose_description => <<EODESC,
 269 Specify guest operating system. This is used to enable special
 270 optimization/features for specific operating systems:
 271
 272 [horizontal]
 273 other;; unspecified OS
 274 wxp;; Microsoft Windows XP
 275 w2k;; Microsoft Windows 2000
 276 w2k3;; Microsoft Windows 2003
 277 w2k8;; Microsoft Windows 2008
 278 wvista;; Microsoft Windows Vista
 279 win7;; Microsoft Windows 7
 280 win8;; Microsoft Windows 8/2012
 281 l24;; Linux 2.4 Kernel
 282 l26;; Linux 2.6/3.X Kernel
 283 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 284 EODESC
 285     },
 286     boot => {
 287         optional => 1,
 288         type => 'string',
 289         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 290         pattern => '[acdn]{1,4}',
 291         default => 'cdn',
 292     },
 293     bootdisk => {
 294         optional => 1,
 295         type => 'string', format => 'pve-qm-bootdisk',
 296         description => "Enable booting from specified disk.",
 297         pattern => '(ide|sata|scsi|virtio)\d+',
 298     },
 299     smp => {
 300         optional => 1,
 301         type => 'integer',
 302         description => "The number of CPUs. Please use option -sockets instead.",
 303         minimum => 1,
 304         default => 1,
 305     },
 306     sockets => {
 307         optional => 1,
 308         type => 'integer',
 309         description => "The number of CPU sockets.",
 310         minimum => 1,
 311         default => 1,
 312     },
 313     cores => {
 314         optional => 1,
 315         type => 'integer',
 316         description => "The number of cores per socket.",
 317         minimum => 1,
 318         default => 1,
 319     },
 320     numa => {
 321         optional => 1,
 322         type => 'boolean',
 323         description => "Enable/disable NUMA.",
 324         default => 0,
 325     },
 326     hugepages => {
 327         optional => 1,
 328         type => 'string',
 329         description => "Enable/disable hugepages memory.",
 330         enum => [qw(any 2 1024)],
 331     },
 332     vcpus => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Number of hotplugged vcpus.",
 336         minimum => 1,
 337         default => 0,
 338     },
 339     acpi => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Enable/disable ACPI.",
 343         default => 1,
 344     },
 345     agent => {
 346         optional => 1,
 347         type => 'boolean',
 348         description => "Enable/disable Qemu GuestAgent.",
 349         default => 0,
 350     },
 351     kvm => {
 352         optional => 1,
 353         type => 'boolean',
 354         description => "Enable/disable KVM hardware virtualization.",
 355         default => 1,
 356     },
 357     tdf => {
 358         optional => 1,
 359         type => 'boolean',
 360         description => "Enable/disable time drift fix.",
 361         default => 0,
 362     },
 363     localtime => {
 364         optional => 1,
 365         type => 'boolean',
 366         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 367     },
 368     freeze => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 372     },
 373     vga => {
 374         optional => 1,
 375         type => 'string',
 376         description => "Select the VGA type.",
 377         verbose_description => "Select the VGA type. If you want to use high resolution" .
 378             " modes (>= 1280x1024x16) then you should use the options " .
 379             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 380             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 381             "display sever. For win* OS you can select how many independent " .
 382             "displays you want, Linux guests can add displays them self. " .
 383             "You can also run without any graphic card, using a serial device" .
 384             " as terminal.",
 385         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 386     },
 387     watchdog => {
 388         optional => 1,
 389         type => 'string', format => 'pve-qm-watchdog',
 390         description => "Create a virtual hardware watchdog device.",
 391         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 392             " (by a guest action), the watchdog must be periodically polled " .
 393             "by an agent inside the guest or else the watchdog will reset " .
 394             "the guest (or execute the respective action specified)",
 395     },
 396     startdate => {
 397         optional => 1,
 398         type => 'string',
 399         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 400         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 401         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 402         default => 'now',
 403     },
 404     startup =>  get_standard_option('pve-startup-order'),
 405     template => {
 406         optional => 1,
 407         type => 'boolean',
 408         description => "Enable/disable Template.",
 409         default => 0,
 410     },
 411     args => {
 412         optional => 1,
 413         type => 'string',
 414         description => "Arbitrary arguments passed to kvm.",
 415         verbose_description => <<EODESCR,
 416 Arbitrary arguments passed to kvm, for example:
 417
 418 args: -no-reboot -no-hpet
 419
 420 NOTE: this option is for experts only.
 421 EODESCR
 422     },
 423     tablet => {
 424         optional => 1,
 425         type => 'boolean',
 426         default => 1,
 427         description => "Enable/disable the USB tablet device.",
 428         verbose_description => "Enable/disable the USB tablet device. This device is " .
 429             "usually needed to allow absolute mouse positioning with VNC. " .
 430             "Else the mouse runs out of sync with normal VNC clients. " .
 431             "If you're running lots of console-only guests on one host, " .
 432             "you may consider disabling this to save some context switches. " .
 433             "This is turned off by default if you use spice (-vga=qxl).",
 434     },
 435     migrate_speed => {
 436         optional => 1,
 437         type => 'integer',
 438         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 439         minimum => 0,
 440         default => 0,
 441     },
 442     migrate_downtime => {
 443         optional => 1,
 444         type => 'number',
 445         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 446         minimum => 0,
 447         default => 0.1,
 448     },
 449     cdrom => {
 450         optional => 1,
 451         type => 'string', format => 'pve-qm-drive',
 452         typetext => 'volume',
 453         description => "This is an alias for option -ide2",
 454     },
 455     cpu => {
 456         optional => 1,
 457         description => "Emulated CPU type.",
 458         type => 'string',
 459         format => $cpu_fmt,
 460     },
 461     parent => get_standard_option('pve-snapshot-name', {
 462         optional => 1,
 463         description => "Parent snapshot name. This is used internally, and should not be modified.",
 464     }),
 465     snaptime => {
 466         optional => 1,
 467         description => "Timestamp for snapshots.",
 468         type => 'integer',
 469         minimum => 0,
 470     },
 471     vmstate => {
 472         optional => 1,
 473         type => 'string', format => 'pve-volume-id',
 474         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 475     },
 476     machine => {
 477         description => "Specific the Qemu machine type.",
 478         type => 'string',
 479         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 480         maxLength => 40,
 481         optional => 1,
 482     },
 483     smbios1 => {
 484         description => "Specify SMBIOS type 1 fields.",
 485         type => 'string', format => 'pve-qm-smbios1',
 486         maxLength => 256,
 487         optional => 1,
 488     },
 489     protection => {
 490         optional => 1,
 491         type => 'boolean',
 492         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 493         default => 0,
 494     },
 495     bios => {
 496         optional => 1,
 497         type => 'string',
 498         enum => [ qw(seabios ovmf) ],
 499         description => "Select BIOS implementation.",
 500         default => 'seabios',
 501     },
 502 };
 503
 504 # what about other qemu settings ?
 505 #cpu => 'string',
 506 #machine => 'string',
 507 #fda => 'file',
 508 #fdb => 'file',
 509 #mtdblock => 'file',
 510 #sd => 'file',
 511 #pflash => 'file',
 512 #snapshot => 'bool',
 513 #bootp => 'file',
 514 ##tftp => 'dir',
 515 ##smb => 'dir',
 516 #kernel => 'file',
 517 #append => 'string',
 518 #initrd => 'file',
 519 ##soundhw => 'string',
 520
 521 while (my ($k, $v) = each %$confdesc) {
 522     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 523 }
 524
 525 my $MAX_IDE_DISKS = 4;
 526 my $MAX_SCSI_DISKS = 14;
 527 my $MAX_VIRTIO_DISKS = 16;
 528 my $MAX_SATA_DISKS = 6;
 529 my $MAX_USB_DEVICES = 5;
 530 my $MAX_NETS = 32;
 531 my $MAX_UNUSED_DISKS = 8;
 532 my $MAX_HOSTPCI_DEVICES = 4;
 533 my $MAX_SERIAL_PORTS = 4;
 534 my $MAX_PARALLEL_PORTS = 3;
 535 my $MAX_NUMA = 8;
 536
 537 my $numa_fmt = {
 538     cpus => {
 539         type => "string",
 540         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 541         description => "CPUs accessing this NUMA node.",
 542         format_description => "id[-id];...",
 543     },
 544     memory => {
 545         type => "number",
 546         description => "Amount of memory this NUMA node provides.",
 547         optional => 1,
 548     },
 549     hostnodes => {
 550         type => "string",
 551         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 552         description => "Host NUMA nodes to use.",
 553         format_description => "id[-id];...",
 554         optional => 1,
 555     },
 556     policy => {
 557         type => 'string',
 558         enum => [qw(preferred bind interleave)],
 559         description => "NUMA allocation policy.",
 560         optional => 1,
 561     },
 562 };
 563 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 564 my $numadesc = {
 565     optional => 1,
 566     type => 'string', format => $numa_fmt,
 567     description => "NUMA topology.",
 568 };
 569 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 570
 571 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 572     $confdesc->{"numa$i"} = $numadesc;
 573 }
 574
 575 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 576                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 577                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 578 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 579
 580 my $net_fmt_bridge_descr = <<__EOD__;
 581 Bridge to attach the network device to. The Proxmox VE standard bridge
 582 is called 'vmbr0'.
 583
 584 If you do not specify a bridge, we create a kvm user (NATed) network
 585 device, which provides DHCP and DNS services. The following addresses
 586 are used:
 587
 588  10.0.2.2   Gateway
 589  10.0.2.3   DNS Server
 590  10.0.2.4   SMB Server
 591
 592 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 593 __EOD__
 594
 595 my $net_fmt = {
 596     macaddr => {
 597         type => 'string',
 598         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 599         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 600         format_description => "XX:XX:XX:XX:XX:XX",
 601         optional => 1,
 602     },
 603     model => {
 604         type => 'string',
 605         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 606         format_description => 'model',
 607         enum => $nic_model_list,
 608         default_key => 1,
 609     },
 610     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 611     bridge => {
 612         type => 'string',
 613         description => $net_fmt_bridge_descr,
 614         format_description => 'bridge',
 615         optional => 1,
 616     },
 617     queues => {
 618         type => 'integer',
 619         minimum => 0, maximum => 16,
 620         description => 'Number of packet queues to be used on the device.',
 621         optional => 1,
 622     },
 623     rate => {
 624         type => 'number',
 625         minimum => 0,
 626         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 627         optional => 1,
 628     },
 629     tag => {
 630         type => 'integer',
 631         minimum => 1, maximum => 4094,
 632         description => 'VLAN tag to apply to packets on this interface.',
 633         optional => 1,
 634     },
 635     trunks => {
 636         type => 'string',
 637         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 638         description => 'VLAN trunks to pass through this interface.',
 639         format_description => 'vlanid[;vlanid...]',
 640         optional => 1,
 641     },
 642     firewall => {
 643         type => 'boolean',
 644         description => 'Whether this interface should be protected by the firewall.',
 645         optional => 1,
 646     },
 647     link_down => {
 648         type => 'boolean',
 649         description => 'Whether this interface should be disconnected (like pulling the plug).',
 650         optional => 1,
 651     },
 652 };
 653
 654 my $netdesc = {
 655     optional => 1,
 656     type => 'string', format => $net_fmt,
 657     description => "Specify network devices.",
 658 };
 659
 660 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 661
 662 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 663     $confdesc->{"net$i"} = $netdesc;
 664 }
 665
 666 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 667 sub verify_volume_id_or_qm_path {
 668     my ($volid, $noerr) = @_;
 669
 670     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 671         return $volid;
 672     }
 673
 674     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 675     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 676     if ($@) {
 677         return undef if $noerr;
 678         die $@;
 679     }
 680     return $volid;
 681 }
 682
 683 my $drivename_hash;
 684
 685 my %drivedesc_base = (
 686     volume => { alias => 'file' },
 687     file => {
 688         type => 'string',
 689         format => 'pve-volume-id-or-qm-path',
 690         default_key => 1,
 691         format_description => 'volume',
 692         description => "The drive's backing volume.",
 693     },
 694     media => {
 695         type => 'string',
 696         enum => [qw(cdrom disk)],
 697         description => "The drive's media type.",
 698         default => 'disk',
 699         optional => 1
 700     },
 701     cyls => {
 702         type => 'integer',
 703         description => "Force the drive's physical geometry to have a specific cylinder count.",
 704         optional => 1
 705     },
 706     heads => {
 707         type => 'integer',
 708         description => "Force the drive's physical geometry to have a specific head count.",
 709         optional => 1
 710     },
 711     secs => {
 712         type => 'integer',
 713         description => "Force the drive's physical geometry to have a specific sector count.",
 714         optional => 1
 715     },
 716     trans => {
 717         type => 'string',
 718         enum => [qw(none lba auto)],
 719         description => "Force disk geometry bios translation mode.",
 720         optional => 1,
 721     },
 722     snapshot => {
 723         type => 'boolean',
 724         description => "Whether the drive should be included when making snapshots.",
 725         optional => 1,
 726     },
 727     cache => {
 728         type => 'string',
 729         enum => [qw(none writethrough writeback unsafe directsync)],
 730         description => "The drive's cache mode",
 731         optional => 1,
 732     },
 733     format => {
 734         type => 'string',
 735         format_description => 'image format',
 736         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 737         description => "The drive's backing file's data format.",
 738         optional => 1,
 739     },
 740     size => {
 741         type => 'string',
 742         format => 'disk-size',
 743         format_description => 'DiskSize',
 744         description => "Disk size. This is purely informational and has no effect.",
 745         optional => 1,
 746     },
 747     backup => {
 748         type => 'boolean',
 749         description => "Whether the drive should be included when making backups.",
 750         optional => 1,
 751     },
 752     werror => {
 753         type => 'string',
 754         enum => [qw(enospc ignore report stop)],
 755         description => 'Write error action.',
 756         optional => 1,
 757     },
 758     aio => {
 759         type => 'string',
 760         enum => [qw(native threads)],
 761         description => 'AIO type to use.',
 762         optional => 1,
 763     },
 764     discard => {
 765         type => 'string',
 766         enum => [qw(ignore on)],
 767         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 768         optional => 1,
 769     },
 770     detect_zeroes => {
 771         type => 'boolean',
 772         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 773         optional => 1,
 774     },
 775     serial => {
 776         type => 'string',
 777         format => 'urlencoded',
 778         format_description => 'serial',
 779         maxLength => 20*3, # *3 since it's %xx url enoded
 780         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 781         optional => 1,
 782     }
 783 );
 784
 785 my %rerror_fmt = (
 786     rerror => {
 787         type => 'string',
 788         enum => [qw(ignore report stop)],
 789         description => 'Read error action.',
 790         optional => 1,
 791     },
 792 );
 793
 794 my %iothread_fmt = ( iothread => {
 795         type => 'boolean',
 796         description => "Whether to use iothreads for this drive",
 797         optional => 1,
 798 });
 799
 800 my %model_fmt = (
 801     model => {
 802         type => 'string',
 803         format => 'urlencoded',
 804         format_description => 'model',
 805         maxLength => 40*3, # *3 since it's %xx url enoded
 806         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 807         optional => 1,
 808     },
 809 );
 810
 811 my %queues_fmt = (
 812     queues => {
 813         type => 'integer',
 814         description => "Number of queues.",
 815         minimum => 2,
 816         optional => 1
 817     }
 818 );
 819
 820 my $add_throttle_desc = sub {
 821     my ($key, $type, $what, $unit, $longunit) = @_;
 822     $drivedesc_base{$key} = {
 823         type => $type,
 824         format_description => $unit,
 825         description => "Maximum $what speed in $longunit per second.",
 826         optional => 1,
 827     };
 828 };
 829 # throughput: (leaky bucket)
 830 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 831 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 832 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 833 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 834 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 835 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 836 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 837 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 838 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 839
 840 # pools: (pool of IO before throttling starts taking effect)
 841 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 842 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 843 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 844 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 845 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 846 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 847
 848 my $ide_fmt = {
 849     %drivedesc_base,
 850     %rerror_fmt,
 851     %model_fmt,
 852 };
 853
 854 my $idedesc = {
 855     optional => 1,
 856     type => 'string', format => $ide_fmt,
 857     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 858 };
 859 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 860
 861 my $scsi_fmt = {
 862     %drivedesc_base,
 863     %iothread_fmt,
 864     %queues_fmt,
 865 };
 866 my $scsidesc = {
 867     optional => 1,
 868     type => 'string', format => $scsi_fmt,
 869     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 870 };
 871 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 872
 873 my $sata_fmt = {
 874     %drivedesc_base,
 875     %rerror_fmt,
 876 };
 877 my $satadesc = {
 878     optional => 1,
 879     type => 'string', format => $sata_fmt,
 880     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 881 };
 882 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 883
 884 my $virtio_fmt = {
 885     %drivedesc_base,
 886     %iothread_fmt,
 887     %rerror_fmt,
 888 };
 889 my $virtiodesc = {
 890     optional => 1,
 891     type => 'string', format => $virtio_fmt,
 892     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 893 };
 894 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 895
 896 my $alldrive_fmt = {
 897     %drivedesc_base,
 898     %rerror_fmt,
 899     %iothread_fmt,
 900     %model_fmt,
 901     %queues_fmt,
 902 };
 903
 904 my $efidisk_fmt = {
 905     volume => { alias => 'file' },
 906     file => {
 907         type => 'string',
 908         format => 'pve-volume-id-or-qm-path',
 909         default_key => 1,
 910         format_description => 'volume',
 911         description => "The drive's backing volume.",
 912     },
 913     format => {
 914         type => 'string',
 915         format_description => 'image format',
 916         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 917         description => "The drive's backing file's data format.",
 918         optional => 1,
 919     },
 920     size => {
 921         type => 'string',
 922         format => 'disk-size',
 923         format_description => 'DiskSize',
 924         description => "Disk size. This is purely informational and has no effect.",
 925         optional => 1,
 926     },
 927 };
 928
 929 my $efidisk_desc = {
 930     optional => 1,
 931     type => 'string', format => $efidisk_fmt,
 932     description => "Configure a Disk for storing EFI vars",
 933 };
 934
 935 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 936
 937 my $usb_fmt = {
 938     host => {
 939         default_key => 1,
 940         type => 'string', format => 'pve-qm-usb-device',
 941         format_description => 'HOSTUSBDEVICE|spice',
 942         description => <<EODESCR,
 943 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 944
 945  'bus-port(.port)*' (decimal numbers) or
 946  'vendor_id:product_id' (hexadeciaml numbers) or
 947  'spice'
 948
 949 You can use the 'lsusb -t' command to list existing usb devices.
 950
 951 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 952
 953 The value 'spice' can be used to add a usb redirection devices for spice.
 954 EODESCR
 955     },
 956     usb3 => {
 957         optional => 1,
 958         type => 'boolean',
 959         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 960         default => 0,
 961     },
 962 };
 963
 964 my $usbdesc = {
 965     optional => 1,
 966     type => 'string', format => $usb_fmt,
 967     description => "Configure an USB device (n is 0 to 4).",
 968 };
 969 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 970
 971 # NOTE: the match-groups of this regex are used in parse_hostpci
 972 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 973 my $hostpci_fmt = {
 974     host => {
 975         default_key => 1,
 976         type => 'string',
 977         pattern => qr/$PCIRE(;$PCIRE)*/,
 978         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 979         description => <<EODESCR,
 980 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 981 of PCI virtual functions of the host. HOSTPCIID syntax is:
 982
 983 'bus:dev.func' (hexadecimal numbers)
 984
 985 You can us the 'lspci' command to list existing PCI devices.
 986 EODESCR
 987     },
 988     rombar => {
 989         type => 'boolean',
 990         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 991         optional => 1,
 992         default => 1,
 993     },
 994     pcie => {
 995         type => 'boolean',
 996         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
 997         optional => 1,
 998         default => 0,
 999     },
1000     'x-vga' => {
1001         type => 'boolean',
1002         description =>  "Enable vfio-vga device support.",
1003         optional => 1,
1004         default => 0,
1005     },
1006 };
1007 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1008
1009 my $hostpcidesc = {
1010         optional => 1,
1011         type => 'string', format => 'pve-qm-hostpci',
1012         description => "Map host PCI devices into guest.",
1013         verbose_description =>  <<EODESCR,
1014 Map host PCI devices into guest.
1015
1016 NOTE: This option allows direct access to host hardware. So it is no longer
1017 possible to migrate such machines - use with special care.
1018
1019 CAUTION: Experimental! User reported problems with this option.
1020 EODESCR
1021 };
1022 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1023
1024 my $serialdesc = {
1025         optional => 1,
1026         type => 'string',
1027         pattern => '(/dev/.+|socket)',
1028         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1029         verbose_description =>  <<EODESCR,
1030 Create a serial device inside the VM (n is 0 to 3), and pass through a
1031 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1032 host side (use 'qm terminal' to open a terminal connection).
1033
1034 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1035
1036 CAUTION: Experimental! User reported problems with this option.
1037 EODESCR
1038 };
1039
1040 my $paralleldesc= {
1041         optional => 1,
1042         type => 'string',
1043         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1044         description =>  "Map host parallel devices (n is 0 to 2).",
1045         verbose_description =>  <<EODESCR,
1046 Map host parallel devices (n is 0 to 2).
1047
1048 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1049
1050 CAUTION: Experimental! User reported problems with this option.
1051 EODESCR
1052 };
1053
1054 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1055     $confdesc->{"parallel$i"} = $paralleldesc;
1056 }
1057
1058 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1059     $confdesc->{"serial$i"} = $serialdesc;
1060 }
1061
1062 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1063     $confdesc->{"hostpci$i"} = $hostpcidesc;
1064 }
1065
1066 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1067     $drivename_hash->{"ide$i"} = 1;
1068     $confdesc->{"ide$i"} = $idedesc;
1069 }
1070
1071 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1072     $drivename_hash->{"sata$i"} = 1;
1073     $confdesc->{"sata$i"} = $satadesc;
1074 }
1075
1076 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1077     $drivename_hash->{"scsi$i"} = 1;
1078     $confdesc->{"scsi$i"} = $scsidesc ;
1079 }
1080
1081 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1082     $drivename_hash->{"virtio$i"} = 1;
1083     $confdesc->{"virtio$i"} = $virtiodesc;
1084 }
1085
1086 $drivename_hash->{efidisk0} = 1;
1087 $confdesc->{efidisk0} = $efidisk_desc;
1088
1089 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1090     $confdesc->{"usb$i"} = $usbdesc;
1091 }
1092
1093 my $unuseddesc = {
1094     optional => 1,
1095     type => 'string', format => 'pve-volume-id',
1096     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1097 };
1098
1099 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1100     $confdesc->{"unused$i"} = $unuseddesc;
1101 }
1102
1103 my $kvm_api_version = 0;
1104
1105 sub kvm_version {
1106
1107     return $kvm_api_version if $kvm_api_version;
1108
1109     my $fh = IO::File->new("</dev/kvm") ||
1110         return 0;
1111
1112     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1113         $kvm_api_version = $v;
1114     }
1115
1116     $fh->close();
1117
1118     return  $kvm_api_version;
1119 }
1120
1121 my $kvm_user_version;
1122
1123 sub kvm_user_version {
1124
1125     return $kvm_user_version if $kvm_user_version;
1126
1127     $kvm_user_version = 'unknown';
1128
1129     my $code = sub {
1130         my $line = shift;
1131         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1132             $kvm_user_version = $2;
1133         }
1134     };
1135
1136     eval { run_command("kvm -version", outfunc => $code); };
1137     warn $@ if $@;
1138
1139     return $kvm_user_version;
1140
1141 }
1142
1143 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1144
1145 sub valid_drive_names {
1146     # order is important - used to autoselect boot disk
1147     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1148             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1149             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1150             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1151             'efidisk0');
1152 }
1153
1154 sub is_valid_drivename {
1155     my $dev = shift;
1156
1157     return defined($drivename_hash->{$dev});
1158 }
1159
1160 sub option_exists {
1161     my $key = shift;
1162     return defined($confdesc->{$key});
1163 }
1164
1165 sub nic_models {
1166     return $nic_model_list;
1167 }
1168
1169 sub os_list_description {
1170
1171     return {
1172         other => 'Other',
1173         wxp => 'Windows XP',
1174         w2k => 'Windows 2000',
1175         w2k3 =>, 'Windows 2003',
1176         w2k8 => 'Windows 2008',
1177         wvista => 'Windows Vista',
1178         win7 => 'Windows 7',
1179         win8 => 'Windows 8/2012',
1180         l24 => 'Linux 2.4',
1181         l26 => 'Linux 2.6',
1182     };
1183 }
1184
1185 my $cdrom_path;
1186
1187 sub get_cdrom_path {
1188
1189     return  $cdrom_path if $cdrom_path;
1190
1191     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1192     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1193     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1194 }
1195
1196 sub get_iso_path {
1197     my ($storecfg, $vmid, $cdrom) = @_;
1198
1199     if ($cdrom eq 'cdrom') {
1200         return get_cdrom_path();
1201     } elsif ($cdrom eq 'none') {
1202         return '';
1203     } elsif ($cdrom =~ m|^/|) {
1204         return $cdrom;
1205     } else {
1206         return PVE::Storage::path($storecfg, $cdrom);
1207     }
1208 }
1209
1210 # try to convert old style file names to volume IDs
1211 sub filename_to_volume_id {
1212     my ($vmid, $file, $media) = @_;
1213
1214     if (!($file eq 'none' || $file eq 'cdrom' ||
1215           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1216
1217         return undef if $file =~ m|/|;
1218
1219         if ($media && $media eq 'cdrom') {
1220             $file = "local:iso/$file";
1221         } else {
1222             $file = "local:$vmid/$file";
1223         }
1224     }
1225
1226     return $file;
1227 }
1228
1229 sub verify_media_type {
1230     my ($opt, $vtype, $media) = @_;
1231
1232     return if !$media;
1233
1234     my $etype;
1235     if ($media eq 'disk') {
1236         $etype = 'images';
1237     } elsif ($media eq 'cdrom') {
1238         $etype = 'iso';
1239     } else {
1240         die "internal error";
1241     }
1242
1243     return if ($vtype eq $etype);
1244
1245     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1246 }
1247
1248 sub cleanup_drive_path {
1249     my ($opt, $storecfg, $drive) = @_;
1250
1251     # try to convert filesystem paths to volume IDs
1252
1253     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1254         ($drive->{file} !~ m|^/dev/.+|) &&
1255         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1256         ($drive->{file} !~ m/^\d+$/)) {
1257         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1258         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1259         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1260         verify_media_type($opt, $vtype, $drive->{media});
1261         $drive->{file} = $volid;
1262     }
1263
1264     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1265 }
1266
1267 sub parse_hotplug_features {
1268     my ($data) = @_;
1269
1270     my $res = {};
1271
1272     return $res if $data eq '0';
1273
1274     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1275
1276     foreach my $feature (PVE::Tools::split_list($data)) {
1277         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1278             $res->{$1} = 1;
1279         } else {
1280             die "invalid hotplug feature '$feature'\n";
1281         }
1282     }
1283     return $res;
1284 }
1285
1286 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1287 sub pve_verify_hotplug_features {
1288     my ($value, $noerr) = @_;
1289
1290     return $value if parse_hotplug_features($value);
1291
1292     return undef if $noerr;
1293
1294     die "unable to parse hotplug option\n";
1295 }
1296
1297 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1298 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1299 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1300 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1301 #        [,iothread=on][,serial=serial][,model=model]
1302
1303 sub parse_drive {
1304     my ($key, $data) = @_;
1305
1306     my ($interface, $index);
1307
1308     if ($key =~ m/^([^\d]+)(\d+)$/) {
1309         $interface = $1;
1310         $index = $2;
1311     } else {
1312         return undef;
1313     }
1314
1315     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1316                                      : $confdesc->{$key}->{format};
1317     if (!$desc) {
1318         warn "invalid drive key: $key\n";
1319         return undef;
1320     }
1321     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1322     return undef if !$res;
1323     $res->{interface} = $interface;
1324     $res->{index} = $index;
1325
1326     my $error = 0;
1327     foreach my $opt (qw(bps bps_rd bps_wr)) {
1328         if (my $bps = defined(delete $res->{$opt})) {
1329             if (defined($res->{"m$opt"})) {
1330                 warn "both $opt and m$opt specified\n";
1331                 ++$error;
1332                 next;
1333             }
1334             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1335         }
1336     }
1337     return undef if $error;
1338
1339     return undef if $res->{mbps_rd} && $res->{mbps};
1340     return undef if $res->{mbps_wr} && $res->{mbps};
1341     return undef if $res->{iops_rd} && $res->{iops};
1342     return undef if $res->{iops_wr} && $res->{iops};
1343
1344     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1345         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1346         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1347         return undef if $res->{interface} eq 'virtio';
1348     }
1349
1350     if (my $size = $res->{size}) {
1351         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1352     }
1353
1354     return $res;
1355 }
1356
1357 sub print_drive {
1358     my ($vmid, $drive) = @_;
1359     my $data = { %$drive };
1360     delete $data->{$_} for qw(index interface);
1361     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1362 }
1363
1364 sub scsi_inquiry {
1365     my($fh, $noerr) = @_;
1366
1367     my $SG_IO = 0x2285;
1368     my $SG_GET_VERSION_NUM = 0x2282;
1369
1370     my $versionbuf = "\x00" x 8;
1371     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1372     if (!$ret) {
1373         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1374         return undef;
1375     }
1376     my $version = unpack("I", $versionbuf);
1377     if ($version < 30000) {
1378         die "scsi generic interface too old\n"  if !$noerr;
1379         return undef;
1380     }
1381
1382     my $buf = "\x00" x 36;
1383     my $sensebuf = "\x00" x 8;
1384     my $cmd = pack("C x3 C x1", 0x12, 36);
1385
1386     # see /usr/include/scsi/sg.h
1387     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1388
1389     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1390                       length($sensebuf), 0, length($buf), $buf,
1391                       $cmd, $sensebuf, 6000);
1392
1393     $ret = ioctl($fh, $SG_IO, $packet);
1394     if (!$ret) {
1395         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1396         return undef;
1397     }
1398
1399     my @res = unpack($sg_io_hdr_t, $packet);
1400     if ($res[17] || $res[18]) {
1401         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1402         return undef;
1403     }
1404
1405     my $res = {};
1406     (my $byte0, my $byte1, $res->{vendor},
1407      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1408
1409     $res->{removable} = $byte1 & 128 ? 1 : 0;
1410     $res->{type} = $byte0 & 31;
1411
1412     return $res;
1413 }
1414
1415 sub path_is_scsi {
1416     my ($path) = @_;
1417
1418     my $fh = IO::File->new("+<$path") || return undef;
1419     my $res = scsi_inquiry($fh, 1);
1420     close($fh);
1421
1422     return $res;
1423 }
1424
1425 sub machine_type_is_q35 {
1426     my ($conf) = @_;
1427
1428     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1429 }
1430
1431 sub print_tabletdevice_full {
1432     my ($conf) = @_;
1433
1434     my $q35 = machine_type_is_q35($conf);
1435
1436     # we use uhci for old VMs because tablet driver was buggy in older qemu
1437     my $usbbus = $q35 ? "ehci" : "uhci";
1438
1439     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1440 }
1441
1442 sub print_drivedevice_full {
1443     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1444
1445     my $device = '';
1446     my $maxdev = 0;
1447
1448     if ($drive->{interface} eq 'virtio') {
1449         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1450         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1451         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1452     } elsif ($drive->{interface} eq 'scsi') {
1453
1454         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1455         my $unit = $drive->{index} % $maxdev;
1456         my $devicetype = 'hd';
1457         my $path = '';
1458         if (drive_is_cdrom($drive)) {
1459             $devicetype = 'cd';
1460         } else {
1461             if ($drive->{file} =~ m|^/|) {
1462                 $path = $drive->{file};
1463                 if (my $info = path_is_scsi($path)) {
1464                     if ($info->{type} == 0) {
1465                         $devicetype = 'block';
1466                     } elsif ($info->{type} == 1) { # tape
1467                         $devicetype = 'generic';
1468                     }
1469                 }
1470             } else {
1471                  $path = PVE::Storage::path($storecfg, $drive->{file});
1472             }
1473
1474             if($path =~ m/^iscsi\:\/\//){
1475                 $devicetype = 'generic';
1476             }
1477         }
1478
1479         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1480             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1481         } else {
1482             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1483         }
1484
1485     } elsif ($drive->{interface} eq 'ide'){
1486         $maxdev = 2;
1487         my $controller = int($drive->{index} / $maxdev);
1488         my $unit = $drive->{index} % $maxdev;
1489         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1490
1491         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1492         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1493             $model = URI::Escape::uri_unescape($model);
1494             $device .= ",model=$model";
1495         }
1496     } elsif ($drive->{interface} eq 'sata'){
1497         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1498         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1499         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1500     } elsif ($drive->{interface} eq 'usb') {
1501         die "implement me";
1502         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1503     } else {
1504         die "unsupported interface type";
1505     }
1506
1507     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1508
1509     return $device;
1510 }
1511
1512 sub get_initiator_name {
1513     my $initiator;
1514
1515     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1516     while (defined(my $line = <$fh>)) {
1517         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1518         $initiator = $1;
1519         last;
1520     }
1521     $fh->close();
1522
1523     return $initiator;
1524 }
1525
1526 sub print_drive_full {
1527     my ($storecfg, $vmid, $drive) = @_;
1528
1529     my $path;
1530     my $volid = $drive->{file};
1531     my $format;
1532
1533     if (drive_is_cdrom($drive)) {
1534         $path = get_iso_path($storecfg, $vmid, $volid);
1535     } else {
1536         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1537         if ($storeid) {
1538             $path = PVE::Storage::path($storecfg, $volid);
1539             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1540             $format = qemu_img_format($scfg, $volname);
1541         } else {
1542             $path = $volid;
1543             $format = "raw";
1544         }
1545    }
1546
1547     my $opts = '';
1548     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1549     foreach my $o (@qemu_drive_options) {
1550         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1551     }
1552     if (my $serial = $drive->{serial}) {
1553         $serial = URI::Escape::uri_unescape($serial);
1554         $opts .= ",serial=$serial";
1555     }
1556
1557     $opts .= ",format=$format" if $format && !$drive->{format};
1558
1559     foreach my $o (qw(bps bps_rd bps_wr)) {
1560         my $v = $drive->{"m$o"};
1561         $opts .= ",$o=" . int($v*1024*1024) if $v;
1562     }
1563
1564     my $cache_direct = 0;
1565
1566     if (my $cache = $drive->{cache}) {
1567         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1568     } elsif (!drive_is_cdrom($drive)) {
1569         $opts .= ",cache=none";
1570         $cache_direct = 1;
1571     }
1572
1573     # aio native works only with O_DIRECT
1574     if (!$drive->{aio}) {
1575         if($cache_direct) {
1576             $opts .= ",aio=native";
1577         } else {
1578             $opts .= ",aio=threads";
1579         }
1580     }
1581
1582     if (!drive_is_cdrom($drive)) {
1583         my $detectzeroes;
1584         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1585             $detectzeroes = 'off';
1586         } elsif ($drive->{discard}) {
1587             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1588         } else {
1589             # This used to be our default with discard not being specified:
1590             $detectzeroes = 'on';
1591         }
1592         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1593     }
1594
1595     my $pathinfo = $path ? "file=$path," : '';
1596
1597     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1598 }
1599
1600 sub print_netdevice_full {
1601     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1602
1603     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1604
1605     my $device = $net->{model};
1606     if ($net->{model} eq 'virtio') {
1607          $device = 'virtio-net-pci';
1608      };
1609
1610     my $pciaddr = print_pci_addr("$netid", $bridges);
1611     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1612     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1613         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1614         my $vectors = $net->{queues} * 2 + 2;
1615         $tmpstr .= ",vectors=$vectors,mq=on";
1616     }
1617     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1618
1619     if ($use_old_bios_files) {
1620         my $romfile;
1621         if ($device eq 'virtio-net-pci') {
1622             $romfile = 'pxe-virtio.rom';
1623         } elsif ($device eq 'e1000') {
1624             $romfile = 'pxe-e1000.rom';
1625         } elsif ($device eq 'ne2k') {
1626             $romfile = 'pxe-ne2k_pci.rom';
1627         } elsif ($device eq 'pcnet') {
1628             $romfile = 'pxe-pcnet.rom';
1629         } elsif ($device eq 'rtl8139') {
1630             $romfile = 'pxe-rtl8139.rom';
1631         }
1632         $tmpstr .= ",romfile=$romfile" if $romfile;
1633     }
1634
1635     return $tmpstr;
1636 }
1637
1638 sub print_netdev_full {
1639     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1640
1641     my $i = '';
1642     if ($netid =~ m/^net(\d+)$/) {
1643         $i = int($1);
1644     }
1645
1646     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1647
1648     my $ifname = "tap${vmid}i$i";
1649
1650     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1651     die "interface name '$ifname' is too long (max 15 character)\n"
1652         if length($ifname) >= 16;
1653
1654     my $vhostparam = '';
1655     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1656
1657     my $vmname = $conf->{name} || "vm$vmid";
1658
1659     my $netdev = "";
1660     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1661
1662     if ($net->{bridge}) {
1663         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1664     } else {
1665         $netdev = "type=user,id=$netid,hostname=$vmname";
1666     }
1667
1668     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1669
1670     return $netdev;
1671 }
1672
1673 sub drive_is_cdrom {
1674     my ($drive) = @_;
1675
1676     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1677
1678 }
1679
1680 sub parse_number_sets {
1681     my ($set) = @_;
1682     my $res = [];
1683     foreach my $part (split(/;/, $set)) {
1684         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1685             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1686             push @$res, [ $1, $2 ];
1687         } else {
1688             die "invalid range: $part\n";
1689         }
1690     }
1691     return $res;
1692 }
1693
1694 sub parse_numa {
1695     my ($data) = @_;
1696
1697     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1698     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1699     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1700     return $res;
1701 }
1702
1703 sub parse_hostpci {
1704     my ($value) = @_;
1705
1706     return undef if !$value;
1707
1708     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1709
1710     my @idlist = split(/;/, $res->{host});
1711     delete $res->{host};
1712     foreach my $id (@idlist) {
1713         if ($id =~ /^$PCIRE$/) {
1714             if (defined($2)) {
1715                 push @{$res->{pciid}}, { id => $1, function => $2 };
1716             } else {
1717                 my $pcidevices = lspci($1);
1718                 $res->{pciid} = $pcidevices->{$1};
1719             }
1720         } else {
1721             # should have been caught by parse_property_string already
1722             die "failed to parse PCI id: $id\n";
1723         }
1724     }
1725     return $res;
1726 }
1727
1728 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1729 sub parse_net {
1730     my ($data) = @_;
1731
1732     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1733     if ($@) {
1734         warn $@;
1735         return undef;
1736     }
1737     if (!defined($res->{macaddr})) {
1738         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1739         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1740     }
1741     return $res;
1742 }
1743
1744 sub print_net {
1745     my $net = shift;
1746
1747     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1748 }
1749
1750 sub add_random_macs {
1751     my ($settings) = @_;
1752
1753     foreach my $opt (keys %$settings) {
1754         next if $opt !~ m/^net(\d+)$/;
1755         my $net = parse_net($settings->{$opt});
1756         next if !$net;
1757         $settings->{$opt} = print_net($net);
1758     }
1759 }
1760
1761 sub vm_is_volid_owner {
1762     my ($storecfg, $vmid, $volid) = @_;
1763
1764     if ($volid !~  m|^/|) {
1765         my ($path, $owner);
1766         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1767         if ($owner && ($owner == $vmid)) {
1768             return 1;
1769         }
1770     }
1771
1772     return undef;
1773 }
1774
1775 sub split_flagged_list {
1776     my $text = shift || '';
1777     $text =~ s/[,;]/ /g;
1778     $text =~ s/^\s+//;
1779     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1780 }
1781
1782 sub join_flagged_list {
1783     my ($how, $lst) = @_;
1784     join $how, map { $lst->{$_} . $_ } keys %$lst;
1785 }
1786
1787 sub vmconfig_delete_pending_option {
1788     my ($conf, $key, $force) = @_;
1789
1790     delete $conf->{pending}->{$key};
1791     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1792     $pending_delete_hash->{$key} = $force ? '!' : '';
1793     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1794 }
1795
1796 sub vmconfig_undelete_pending_option {
1797     my ($conf, $key) = @_;
1798
1799     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1800     delete $pending_delete_hash->{$key};
1801
1802     if (%$pending_delete_hash) {
1803         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1804     } else {
1805         delete $conf->{pending}->{delete};
1806     }
1807 }
1808
1809 sub vmconfig_register_unused_drive {
1810     my ($storecfg, $vmid, $conf, $drive) = @_;
1811
1812     if (!drive_is_cdrom($drive)) {
1813         my $volid = $drive->{file};
1814         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1815             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1816         }
1817     }
1818 }
1819
1820 sub vmconfig_cleanup_pending {
1821     my ($conf) = @_;
1822
1823     # remove pending changes when nothing changed
1824     my $changes;
1825     foreach my $opt (keys %{$conf->{pending}}) {
1826         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1827             $changes = 1;
1828             delete $conf->{pending}->{$opt};
1829         }
1830     }
1831
1832     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1833     my $pending_delete_hash = {};
1834     while (my ($opt, $force) = each %$current_delete_hash) {
1835         if (defined($conf->{$opt})) {
1836             $pending_delete_hash->{$opt} = $force;
1837         } else {
1838             $changes = 1;
1839         }
1840     }
1841
1842     if (%$pending_delete_hash) {
1843         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1844     } else {
1845         delete $conf->{pending}->{delete};
1846     }
1847
1848     return $changes;
1849 }
1850
1851 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1852 my $smbios1_fmt = {
1853     uuid => {
1854         type => 'string',
1855         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1856         format_description => 'UUID',
1857         description => "Set SMBIOS1 UUID.",
1858         optional => 1,
1859     },
1860     version => {
1861         type => 'string',
1862         pattern => '\S+',
1863         format_description => 'string',
1864         description => "Set SMBIOS1 version.",
1865         optional => 1,
1866     },
1867     serial => {
1868         type => 'string',
1869         pattern => '\S+',
1870         format_description => 'string',
1871         description => "Set SMBIOS1 serial number.",
1872         optional => 1,
1873     },
1874     manufacturer => {
1875         type => 'string',
1876         pattern => '\S+',
1877         format_description => 'string',
1878         description => "Set SMBIOS1 manufacturer.",
1879         optional => 1,
1880     },
1881     product => {
1882         type => 'string',
1883         pattern => '\S+',
1884         format_description => 'string',
1885         description => "Set SMBIOS1 product ID.",
1886         optional => 1,
1887     },
1888     sku => {
1889         type => 'string',
1890         pattern => '\S+',
1891         format_description => 'string',
1892         description => "Set SMBIOS1 SKU string.",
1893         optional => 1,
1894     },
1895     family => {
1896         type => 'string',
1897         pattern => '\S+',
1898         format_description => 'string',
1899         description => "Set SMBIOS1 family string.",
1900         optional => 1,
1901     },
1902 };
1903
1904 sub parse_smbios1 {
1905     my ($data) = @_;
1906
1907     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1908     warn $@ if $@;
1909     return $res;
1910 }
1911
1912 sub print_smbios1 {
1913     my ($smbios1) = @_;
1914     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1915 }
1916
1917 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1918
1919 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1920 sub verify_bootdisk {
1921     my ($value, $noerr) = @_;
1922
1923     return $value if is_valid_drivename($value);
1924
1925     return undef if $noerr;
1926
1927     die "invalid boot disk '$value'\n";
1928 }
1929
1930 sub parse_watchdog {
1931     my ($value) = @_;
1932
1933     return undef if !$value;
1934
1935     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1936     warn $@ if $@;
1937     return $res;
1938 }
1939
1940 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1941 sub verify_usb_device {
1942     my ($value, $noerr) = @_;
1943
1944     return $value if parse_usb_device($value);
1945
1946     return undef if $noerr;
1947
1948     die "unable to parse usb device\n";
1949 }
1950
1951 # add JSON properties for create and set function
1952 sub json_config_properties {
1953     my $prop = shift;
1954
1955     foreach my $opt (keys %$confdesc) {
1956         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1957         $prop->{$opt} = $confdesc->{$opt};
1958     }
1959
1960     return $prop;
1961 }
1962
1963 sub check_type {
1964     my ($key, $value) = @_;
1965
1966     die "unknown setting '$key'\n" if !$confdesc->{$key};
1967
1968     my $type = $confdesc->{$key}->{type};
1969
1970     if (!defined($value)) {
1971         die "got undefined value\n";
1972     }
1973
1974     if ($value =~ m/[\n\r]/) {
1975         die "property contains a line feed\n";
1976     }
1977
1978     if ($type eq 'boolean') {
1979         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1980         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1981         die "type check ('boolean') failed - got '$value'\n";
1982     } elsif ($type eq 'integer') {
1983         return int($1) if $value =~ m/^(\d+)$/;
1984         die "type check ('integer') failed - got '$value'\n";
1985     } elsif ($type eq 'number') {
1986         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1987         die "type check ('number') failed - got '$value'\n";
1988     } elsif ($type eq 'string') {
1989         if (my $fmt = $confdesc->{$key}->{format}) {
1990             if ($fmt eq 'pve-qm-drive') {
1991                 # special case - we need to pass $key to parse_drive()
1992                 my $drive = parse_drive($key, $value);
1993                 return $value if $drive;
1994                 die "unable to parse drive options\n";
1995             }
1996             PVE::JSONSchema::check_format($fmt, $value);
1997             return $value;
1998         }
1999         $value =~ s/^\"(.*)\"$/$1/;
2000         return $value;
2001     } else {
2002         die "internal error"
2003     }
2004 }
2005
2006 sub check_iommu_support{
2007     #fixme : need to check IOMMU support
2008     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2009
2010     my $iommu=1;
2011     return $iommu;
2012
2013 }
2014
2015 sub touch_config {
2016     my ($vmid) = @_;
2017
2018     my $conf = PVE::QemuConfig->config_file($vmid);
2019     utime undef, undef, $conf;
2020 }
2021
2022 sub destroy_vm {
2023     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2024
2025     my $conffile = PVE::QemuConfig->config_file($vmid);
2026
2027     my $conf = PVE::QemuConfig->load_config($vmid);
2028
2029     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2030
2031     # only remove disks owned by this VM
2032     foreach_drive($conf, sub {
2033         my ($ds, $drive) = @_;
2034
2035         return if drive_is_cdrom($drive);
2036
2037         my $volid = $drive->{file};
2038
2039         return if !$volid || $volid =~ m|^/|;
2040
2041         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2042         return if !$path || !$owner || ($owner != $vmid);
2043
2044         PVE::Storage::vdisk_free($storecfg, $volid);
2045     });
2046
2047     if ($keep_empty_config) {
2048         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2049     } else {
2050         unlink $conffile;
2051     }
2052
2053     # also remove unused disk
2054     eval {
2055         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2056
2057         eval {
2058             PVE::Storage::foreach_volid($dl, sub {
2059                 my ($volid, $sid, $volname, $d) = @_;
2060                 PVE::Storage::vdisk_free($storecfg, $volid);
2061             });
2062         };
2063         warn $@ if $@;
2064
2065     };
2066     warn $@ if $@;
2067 }
2068
2069 sub parse_vm_config {
2070     my ($filename, $raw) = @_;
2071
2072     return undef if !defined($raw);
2073
2074     my $res = {
2075         digest => Digest::SHA::sha1_hex($raw),
2076         snapshots => {},
2077         pending => {},
2078     };
2079
2080     $filename =~ m|/qemu-server/(\d+)\.conf$|
2081         || die "got strange filename '$filename'";
2082
2083     my $vmid = $1;
2084
2085     my $conf = $res;
2086     my $descr;
2087     my $section = '';
2088
2089     my @lines = split(/\n/, $raw);
2090     foreach my $line (@lines) {
2091         next if $line =~ m/^\s*$/;
2092
2093         if ($line =~ m/^\[PENDING\]\s*$/i) {
2094             $section = 'pending';
2095             if (defined($descr)) {
2096                 $descr =~ s/\s+$//;
2097                 $conf->{description} = $descr;
2098             }
2099             $descr = undef;
2100             $conf = $res->{$section} = {};
2101             next;
2102
2103         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2104             $section = $1;
2105             if (defined($descr)) {
2106                 $descr =~ s/\s+$//;
2107                 $conf->{description} = $descr;
2108             }
2109             $descr = undef;
2110             $conf = $res->{snapshots}->{$section} = {};
2111             next;
2112         }
2113
2114         if ($line =~ m/^\#(.*)\s*$/) {
2115             $descr = '' if !defined($descr);
2116             $descr .= PVE::Tools::decode_text($1) . "\n";
2117             next;
2118         }
2119
2120         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2121             $descr = '' if !defined($descr);
2122             $descr .= PVE::Tools::decode_text($2);
2123         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2124             $conf->{snapstate} = $1;
2125         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2126             my $key = $1;
2127             my $value = $2;
2128             $conf->{$key} = $value;
2129         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2130             my $value = $1;
2131             if ($section eq 'pending') {
2132                 $conf->{delete} = $value; # we parse this later
2133             } else {
2134                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2135             }
2136         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2137             my $key = $1;
2138             my $value = $2;
2139             eval { $value = check_type($key, $value); };
2140             if ($@) {
2141                 warn "vm $vmid - unable to parse value of '$key' - $@";
2142             } else {
2143                 my $fmt = $confdesc->{$key}->{format};
2144                 if ($fmt && $fmt eq 'pve-qm-drive') {
2145                     my $v = parse_drive($key, $value);
2146                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2147                         $v->{file} = $volid;
2148                         $value = print_drive($vmid, $v);
2149                     } else {
2150                         warn "vm $vmid - unable to parse value of '$key'\n";
2151                         next;
2152                     }
2153                 }
2154
2155                 if ($key eq 'cdrom') {
2156                     $conf->{ide2} = $value;
2157                 } else {
2158                     $conf->{$key} = $value;
2159                 }
2160             }
2161         }
2162     }
2163
2164     if (defined($descr)) {
2165         $descr =~ s/\s+$//;
2166         $conf->{description} = $descr;
2167     }
2168     delete $res->{snapstate}; # just to be sure
2169
2170     return $res;
2171 }
2172
2173 sub write_vm_config {
2174     my ($filename, $conf) = @_;
2175
2176     delete $conf->{snapstate}; # just to be sure
2177
2178     if ($conf->{cdrom}) {
2179         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2180         $conf->{ide2} = $conf->{cdrom};
2181         delete $conf->{cdrom};
2182     }
2183
2184     # we do not use 'smp' any longer
2185     if ($conf->{sockets}) {
2186         delete $conf->{smp};
2187     } elsif ($conf->{smp}) {
2188         $conf->{sockets} = $conf->{smp};
2189         delete $conf->{cores};
2190         delete $conf->{smp};
2191     }
2192
2193     my $used_volids = {};
2194
2195     my $cleanup_config = sub {
2196         my ($cref, $pending, $snapname) = @_;
2197
2198         foreach my $key (keys %$cref) {
2199             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2200                 $key eq 'snapstate' || $key eq 'pending';
2201             my $value = $cref->{$key};
2202             if ($key eq 'delete') {
2203                 die "propertry 'delete' is only allowed in [PENDING]\n"
2204                     if !$pending;
2205                 # fixme: check syntax?
2206                 next;
2207             }
2208             eval { $value = check_type($key, $value); };
2209             die "unable to parse value of '$key' - $@" if $@;
2210
2211             $cref->{$key} = $value;
2212
2213             if (!$snapname && is_valid_drivename($key)) {
2214                 my $drive = parse_drive($key, $value);
2215                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2216             }
2217         }
2218     };
2219
2220     &$cleanup_config($conf);
2221
2222     &$cleanup_config($conf->{pending}, 1);
2223
2224     foreach my $snapname (keys %{$conf->{snapshots}}) {
2225         die "internal error" if $snapname eq 'pending';
2226         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2227     }
2228
2229     # remove 'unusedX' settings if we re-add a volume
2230     foreach my $key (keys %$conf) {
2231         my $value = $conf->{$key};
2232         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2233             delete $conf->{$key};
2234         }
2235     }
2236
2237     my $generate_raw_config = sub {
2238         my ($conf, $pending) = @_;
2239
2240         my $raw = '';
2241
2242         # add description as comment to top of file
2243         if (defined(my $descr = $conf->{description})) {
2244             if ($descr) {
2245                 foreach my $cl (split(/\n/, $descr)) {
2246                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2247                 }
2248             } else {
2249                 $raw .= "#\n" if $pending;
2250             }
2251         }
2252
2253         foreach my $key (sort keys %$conf) {
2254             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2255             $raw .= "$key: $conf->{$key}\n";
2256         }
2257         return $raw;
2258     };
2259
2260     my $raw = &$generate_raw_config($conf);
2261
2262     if (scalar(keys %{$conf->{pending}})){
2263         $raw .= "\n[PENDING]\n";
2264         $raw .= &$generate_raw_config($conf->{pending}, 1);
2265     }
2266
2267     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2268         $raw .= "\n[$snapname]\n";
2269         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2270     }
2271
2272     return $raw;
2273 }
2274
2275 sub load_defaults {
2276
2277     my $res = {};
2278
2279     # we use static defaults from our JSON schema configuration
2280     foreach my $key (keys %$confdesc) {
2281         if (defined(my $default = $confdesc->{$key}->{default})) {
2282             $res->{$key} = $default;
2283         }
2284     }
2285
2286     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2287     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2288
2289     return $res;
2290 }
2291
2292 sub config_list {
2293     my $vmlist = PVE::Cluster::get_vmlist();
2294     my $res = {};
2295     return $res if !$vmlist || !$vmlist->{ids};
2296     my $ids = $vmlist->{ids};
2297
2298     foreach my $vmid (keys %$ids) {
2299         my $d = $ids->{$vmid};
2300         next if !$d->{node} || $d->{node} ne $nodename;
2301         next if !$d->{type} || $d->{type} ne 'qemu';
2302         $res->{$vmid}->{exists} = 1;
2303     }
2304     return $res;
2305 }
2306
2307 # test if VM uses local resources (to prevent migration)
2308 sub check_local_resources {
2309     my ($conf, $noerr) = @_;
2310
2311     my $loc_res = 0;
2312
2313     $loc_res = 1 if $conf->{hostusb}; # old syntax
2314     $loc_res = 1 if $conf->{hostpci}; # old syntax
2315
2316     foreach my $k (keys %$conf) {
2317         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2318         # sockets are safe: they will recreated be on the target side post-migrate
2319         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2320         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2321     }
2322
2323     die "VM uses local resources\n" if $loc_res && !$noerr;
2324
2325     return $loc_res;
2326 }
2327
2328 # check if used storages are available on all nodes (use by migrate)
2329 sub check_storage_availability {
2330     my ($storecfg, $conf, $node) = @_;
2331
2332     foreach_drive($conf, sub {
2333         my ($ds, $drive) = @_;
2334
2335         my $volid = $drive->{file};
2336         return if !$volid;
2337
2338         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2339         return if !$sid;
2340
2341         # check if storage is available on both nodes
2342         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2343         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2344    });
2345 }
2346
2347 # list nodes where all VM images are available (used by has_feature API)
2348 sub shared_nodes {
2349     my ($conf, $storecfg) = @_;
2350
2351     my $nodelist = PVE::Cluster::get_nodelist();
2352     my $nodehash = { map { $_ => 1 } @$nodelist };
2353     my $nodename = PVE::INotify::nodename();
2354
2355     foreach_drive($conf, sub {
2356         my ($ds, $drive) = @_;
2357
2358         my $volid = $drive->{file};
2359         return if !$volid;
2360
2361         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2362         if ($storeid) {
2363             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2364             if ($scfg->{disable}) {
2365                 $nodehash = {};
2366             } elsif (my $avail = $scfg->{nodes}) {
2367                 foreach my $node (keys %$nodehash) {
2368                     delete $nodehash->{$node} if !$avail->{$node};
2369                 }
2370             } elsif (!$scfg->{shared}) {
2371                 foreach my $node (keys %$nodehash) {
2372                     delete $nodehash->{$node} if $node ne $nodename
2373                 }
2374             }
2375         }
2376     });
2377
2378     return $nodehash
2379 }
2380
2381 sub check_cmdline {
2382     my ($pidfile, $pid) = @_;
2383
2384     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2385     if (defined($fh)) {
2386         my $line = <$fh>;
2387         $fh->close;
2388         return undef if !$line;
2389         my @param = split(/\0/, $line);
2390
2391         my $cmd = $param[0];
2392         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2393
2394         for (my $i = 0; $i < scalar (@param); $i++) {
2395             my $p = $param[$i];
2396             next if !$p;
2397             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2398                 my $p = $param[$i+1];
2399                 return 1 if $p && ($p eq $pidfile);
2400                 return undef;
2401             }
2402         }
2403     }
2404     return undef;
2405 }
2406
2407 sub check_running {
2408     my ($vmid, $nocheck, $node) = @_;
2409
2410     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2411
2412     die "unable to find configuration file for VM $vmid - no such machine\n"
2413         if !$nocheck && ! -f $filename;
2414
2415     my $pidfile = pidfile_name($vmid);
2416
2417     if (my $fd = IO::File->new("<$pidfile")) {
2418         my $st = stat($fd);
2419         my $line = <$fd>;
2420         close($fd);
2421
2422         my $mtime = $st->mtime;
2423         if ($mtime > time()) {
2424             warn "file '$filename' modified in future\n";
2425         }
2426
2427         if ($line =~ m/^(\d+)$/) {
2428             my $pid = $1;
2429             if (check_cmdline($pidfile, $pid)) {
2430                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2431                     return $pid;
2432                 }
2433             }
2434         }
2435     }
2436
2437     return undef;
2438 }
2439
2440 sub vzlist {
2441
2442     my $vzlist = config_list();
2443
2444     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2445
2446     while (defined(my $de = $fd->read)) {
2447         next if $de !~ m/^(\d+)\.pid$/;
2448         my $vmid = $1;
2449         next if !defined($vzlist->{$vmid});
2450         if (my $pid = check_running($vmid)) {
2451             $vzlist->{$vmid}->{pid} = $pid;
2452         }
2453     }
2454
2455     return $vzlist;
2456 }
2457
2458 sub disksize {
2459     my ($storecfg, $conf) = @_;
2460
2461     my $bootdisk = $conf->{bootdisk};
2462     return undef if !$bootdisk;
2463     return undef if !is_valid_drivename($bootdisk);
2464
2465     return undef if !$conf->{$bootdisk};
2466
2467     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2468     return undef if !defined($drive);
2469
2470     return undef if drive_is_cdrom($drive);
2471
2472     my $volid = $drive->{file};
2473     return undef if !$volid;
2474
2475     return $drive->{size};
2476 }
2477
2478 my $last_proc_pid_stat;
2479
2480 # get VM status information
2481 # This must be fast and should not block ($full == false)
2482 # We only query KVM using QMP if $full == true (this can be slow)
2483 sub vmstatus {
2484     my ($opt_vmid, $full) = @_;
2485
2486     my $res = {};
2487
2488     my $storecfg = PVE::Storage::config();
2489
2490     my $list = vzlist();
2491     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2492
2493     my $cpucount = $cpuinfo->{cpus} || 1;
2494
2495     foreach my $vmid (keys %$list) {
2496         next if $opt_vmid && ($vmid ne $opt_vmid);
2497
2498         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2499         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2500
2501         my $d = {};
2502         $d->{pid} = $list->{$vmid}->{pid};
2503
2504         # fixme: better status?
2505         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2506
2507         my $size = disksize($storecfg, $conf);
2508         if (defined($size)) {
2509             $d->{disk} = 0; # no info available
2510             $d->{maxdisk} = $size;
2511         } else {
2512             $d->{disk} = 0;
2513             $d->{maxdisk} = 0;
2514         }
2515
2516         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2517         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2518         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2519
2520         $d->{name} = $conf->{name} || "VM $vmid";
2521         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2522
2523         if ($conf->{balloon}) {
2524             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2525             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2526         }
2527
2528         $d->{uptime} = 0;
2529         $d->{cpu} = 0;
2530         $d->{mem} = 0;
2531
2532         $d->{netout} = 0;
2533         $d->{netin} = 0;
2534
2535         $d->{diskread} = 0;
2536         $d->{diskwrite} = 0;
2537
2538         $d->{template} = PVE::QemuConfig->is_template($conf);
2539
2540         $res->{$vmid} = $d;
2541     }
2542
2543     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2544     foreach my $dev (keys %$netdev) {
2545         next if $dev !~ m/^tap([1-9]\d*)i/;
2546         my $vmid = $1;
2547         my $d = $res->{$vmid};
2548         next if !$d;
2549
2550         $d->{netout} += $netdev->{$dev}->{receive};
2551         $d->{netin} += $netdev->{$dev}->{transmit};
2552
2553         if ($full) {
2554             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2555             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2556         }
2557
2558     }
2559
2560     my $ctime = gettimeofday;
2561
2562     foreach my $vmid (keys %$list) {
2563
2564         my $d = $res->{$vmid};
2565         my $pid = $d->{pid};
2566         next if !$pid;
2567
2568         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2569         next if !$pstat; # not running
2570
2571         my $used = $pstat->{utime} + $pstat->{stime};
2572
2573         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2574
2575         if ($pstat->{vsize}) {
2576             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2577         }
2578
2579         my $old = $last_proc_pid_stat->{$pid};
2580         if (!$old) {
2581             $last_proc_pid_stat->{$pid} = {
2582                 time => $ctime,
2583                 used => $used,
2584                 cpu => 0,
2585             };
2586             next;
2587         }
2588
2589         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2590
2591         if ($dtime > 1000) {
2592             my $dutime = $used -  $old->{used};
2593
2594             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2595             $last_proc_pid_stat->{$pid} = {
2596                 time => $ctime,
2597                 used => $used,
2598                 cpu => $d->{cpu},
2599             };
2600         } else {
2601             $d->{cpu} = $old->{cpu};
2602         }
2603     }
2604
2605     return $res if !$full;
2606
2607     my $qmpclient = PVE::QMPClient->new();
2608
2609     my $ballooncb = sub {
2610         my ($vmid, $resp) = @_;
2611
2612         my $info = $resp->{'return'};
2613         return if !$info->{max_mem};
2614
2615         my $d = $res->{$vmid};
2616
2617         # use memory assigned to VM
2618         $d->{maxmem} = $info->{max_mem};
2619         $d->{balloon} = $info->{actual};
2620
2621         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2622             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2623             $d->{freemem} = $info->{free_mem};
2624         }
2625
2626         $d->{ballooninfo} = $info;
2627     };
2628
2629     my $blockstatscb = sub {
2630         my ($vmid, $resp) = @_;
2631         my $data = $resp->{'return'} || [];
2632         my $totalrdbytes = 0;
2633         my $totalwrbytes = 0;
2634
2635         for my $blockstat (@$data) {
2636             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2637             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2638
2639             $blockstat->{device} =~ s/drive-//;
2640             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2641         }
2642         $res->{$vmid}->{diskread} = $totalrdbytes;
2643         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2644     };
2645
2646     my $statuscb = sub {
2647         my ($vmid, $resp) = @_;
2648
2649         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2650         # this fails if ballon driver is not loaded, so this must be
2651         # the last commnand (following command are aborted if this fails).
2652         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2653
2654         my $status = 'unknown';
2655         if (!defined($status = $resp->{'return'}->{status})) {
2656             warn "unable to get VM status\n";
2657             return;
2658         }
2659
2660         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2661     };
2662
2663     foreach my $vmid (keys %$list) {
2664         next if $opt_vmid && ($vmid ne $opt_vmid);
2665         next if !$res->{$vmid}->{pid}; # not running
2666         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2667     }
2668
2669     $qmpclient->queue_execute(undef, 1);
2670
2671     foreach my $vmid (keys %$list) {
2672         next if $opt_vmid && ($vmid ne $opt_vmid);
2673         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2674     }
2675
2676     return $res;
2677 }
2678
2679 sub foreach_drive {
2680     my ($conf, $func, @param) = @_;
2681
2682     foreach my $ds (valid_drive_names()) {
2683         next if !defined($conf->{$ds});
2684
2685         my $drive = parse_drive($ds, $conf->{$ds});
2686         next if !$drive;
2687
2688         &$func($ds, $drive, @param);
2689     }
2690 }
2691
2692 sub foreach_volid {
2693     my ($conf, $func, @param) = @_;
2694
2695     my $volhash = {};
2696
2697     my $test_volid = sub {
2698         my ($volid, $is_cdrom) = @_;
2699
2700         return if !$volid;
2701
2702         $volhash->{$volid} = $is_cdrom || 0;
2703     };
2704
2705     foreach_drive($conf, sub {
2706         my ($ds, $drive) = @_;
2707         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2708     });
2709
2710     foreach my $snapname (keys %{$conf->{snapshots}}) {
2711         my $snap = $conf->{snapshots}->{$snapname};
2712         &$test_volid($snap->{vmstate}, 0);
2713         foreach_drive($snap, sub {
2714             my ($ds, $drive) = @_;
2715             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2716         });
2717     }
2718
2719     foreach my $volid (keys %$volhash) {
2720         &$func($volid, $volhash->{$volid}, @param);
2721     }
2722 }
2723
2724 sub vga_conf_has_spice {
2725     my ($vga) = @_;
2726
2727     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2728
2729     return $1 || 1;
2730 }
2731
2732 sub config_to_command {
2733     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2734
2735     my $cmd = [];
2736     my $globalFlags = [];
2737     my $machineFlags = [];
2738     my $rtcFlags = [];
2739     my $cpuFlags = [];
2740     my $devices = [];
2741     my $pciaddr = '';
2742     my $bridges = {};
2743     my $kvmver = kvm_user_version();
2744     my $vernum = 0; # unknown
2745     my $ostype = $conf->{ostype};
2746     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2747         $vernum = $1*1000000+$2*1000;
2748     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2749         $vernum = $1*1000000+$2*1000+$3;
2750     }
2751
2752     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2753
2754     my $have_ovz = -f '/proc/vz/vestat';
2755
2756     my $q35 = machine_type_is_q35($conf);
2757     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2758     my $machine_type = $forcemachine || $conf->{machine};
2759     my $use_old_bios_files = undef;
2760     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2761
2762     my $cpuunits = defined($conf->{cpuunits}) ?
2763             $conf->{cpuunits} : $defaults->{cpuunits};
2764
2765     push @$cmd, '/usr/bin/kvm';
2766
2767     push @$cmd, '-id', $vmid;
2768
2769     my $use_virtio = 0;
2770
2771     my $qmpsocket = qmp_socket($vmid);
2772     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2773     push @$cmd, '-mon', "chardev=qmp,mode=control";
2774
2775
2776     push @$cmd, '-pidfile' , pidfile_name($vmid);
2777
2778     push @$cmd, '-daemonize';
2779
2780     if ($conf->{smbios1}) {
2781         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2782     }
2783
2784     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2785         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2786         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2787         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2788         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2789         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2790         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2791     }
2792
2793
2794     # add usb controllers
2795     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2796     push @$devices, @usbcontrollers if @usbcontrollers;
2797     my $vga = $conf->{vga};
2798
2799     my $qxlnum = vga_conf_has_spice($vga);
2800     $vga = 'qxl' if $qxlnum;
2801
2802     if (!$vga) {
2803         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2804                                 $conf->{ostype} eq 'win7' ||
2805                                 $conf->{ostype} eq 'w2k8')) {
2806             $vga = 'std';
2807         } else {
2808             $vga = 'cirrus';
2809         }
2810     }
2811
2812     # enable absolute mouse coordinates (needed by vnc)
2813     my $tablet;
2814     if (defined($conf->{tablet})) {
2815         $tablet = $conf->{tablet};
2816     } else {
2817         $tablet = $defaults->{tablet};
2818         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2819         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2820     }
2821
2822     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2823
2824     my $kvm_off = 0;
2825     # host pci devices
2826     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2827         my $d = parse_hostpci($conf->{"hostpci$i"});
2828         next if !$d;
2829
2830         my $pcie = $d->{pcie};
2831         if($pcie){
2832             die "q35 machine model is not enabled" if !$q35;
2833             $pciaddr = print_pcie_addr("hostpci$i");
2834         }else{
2835             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2836         }
2837
2838         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2839         my $xvga = '';
2840         if ($d->{'x-vga'}) {
2841             $xvga = ',x-vga=on';
2842             $kvm_off = 1;
2843             $vga = 'none';
2844             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2845                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2846             }
2847             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2848                 $xvga = "";
2849             }
2850         }
2851         my $pcidevices = $d->{pciid};
2852         my $multifunction = 1 if @$pcidevices > 1;
2853
2854         my $j=0;
2855         foreach my $pcidevice (@$pcidevices) {
2856
2857             my $id = "hostpci$i";
2858             $id .= ".$j" if $multifunction;
2859             my $addr = $pciaddr;
2860             $addr .= ".$j" if $multifunction;
2861             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2862
2863             if($j == 0){
2864                 $devicestr .= "$rombar$xvga";
2865                 $devicestr .= ",multifunction=on" if $multifunction;
2866             }
2867
2868             push @$devices, '-device', $devicestr;
2869             $j++;
2870         }
2871     }
2872
2873     # usb devices
2874     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2875     push @$devices, @usbdevices if @usbdevices;
2876     # serial devices
2877     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2878         if (my $path = $conf->{"serial$i"}) {
2879             if ($path eq 'socket') {
2880                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2881                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2882                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2883             } else {
2884                 die "no such serial device\n" if ! -c $path;
2885                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2886                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2887             }
2888         }
2889     }
2890
2891     # parallel devices
2892     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2893         if (my $path = $conf->{"parallel$i"}) {
2894             die "no such parallel device\n" if ! -c $path;
2895             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2896             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2897             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2898         }
2899     }
2900
2901     my $vmname = $conf->{name} || "vm$vmid";
2902
2903     push @$cmd, '-name', $vmname;
2904
2905     my $sockets = 1;
2906     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2907     $sockets = $conf->{sockets} if  $conf->{sockets};
2908
2909     my $cores = $conf->{cores} || 1;
2910
2911     my $maxcpus = $sockets * $cores;
2912
2913     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2914
2915     my $allowed_vcpus = $cpuinfo->{cpus};
2916
2917     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2918         if ($allowed_vcpus < $maxcpus);
2919
2920     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2921
2922     push @$cmd, '-nodefaults';
2923
2924     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2925
2926     my $bootindex_hash = {};
2927     my $i = 1;
2928     foreach my $o (split(//, $bootorder)) {
2929         $bootindex_hash->{$o} = $i*100;
2930         $i++;
2931     }
2932
2933     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2934
2935     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2936
2937     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2938
2939     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2940
2941     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2942         my $socket = vnc_socket($vmid);
2943         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2944     } else {
2945         push @$cmd, '-nographic';
2946     }
2947
2948     # time drift fix
2949     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2950
2951     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2952     my $useLocaltime = $conf->{localtime};
2953
2954     if ($ostype) {
2955         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2956
2957         if ($ostype =~ m/^w/) { # windows
2958             $useLocaltime = 1 if !defined($conf->{localtime});
2959
2960             # use time drift fix when acpi is enabled
2961             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2962                 $tdf = 1 if !defined($conf->{tdf});
2963             }
2964         }
2965
2966         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2967             $ostype eq 'wvista') {
2968             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2969             push @$cmd, '-no-hpet';
2970             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2971                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2972                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2973                 push @$cpuFlags , 'hv_time' if !$nokvm;
2974
2975                 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
2976                     push @$cpuFlags , 'hv_reset' if !$nokvm;
2977                     push @$cpuFlags , 'hv_vpindex' if !$nokvm;
2978                     push @$cpuFlags , 'hv_runtime' if !$nokvm;
2979                 }
2980
2981             } else {
2982                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2983             }
2984         }
2985
2986         if ($ostype eq 'win7' || $ostype eq 'win8') {
2987             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2988         }
2989     }
2990
2991     push @$rtcFlags, 'driftfix=slew' if $tdf;
2992
2993     if ($nokvm) {
2994         push @$machineFlags, 'accel=tcg';
2995     } else {
2996         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2997     }
2998
2999     if ($machine_type) {
3000         push @$machineFlags, "type=${machine_type}";
3001     }
3002
3003     if ($conf->{startdate}) {
3004         push @$rtcFlags, "base=$conf->{startdate}";
3005     } elsif ($useLocaltime) {
3006         push @$rtcFlags, 'base=localtime';
3007     }
3008
3009     my $cpu = $nokvm ? "qemu64" : "kvm64";
3010     if (my $cputype = $conf->{cpu}) {
3011         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3012             or die "Cannot parse cpu description: $cputype\n";
3013         $cpu = $cpuconf->{cputype};
3014         $kvm_off = 1 if $cpuconf->{hidden};
3015     }
3016
3017     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3018
3019     push @$cpuFlags , '-x2apic'
3020         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3021
3022     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3023
3024     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3025
3026     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3027
3028         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3029         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3030     }
3031
3032     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3033
3034     push @$cpuFlags, 'kvm=off' if $kvm_off;
3035
3036     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3037         die "internal error"; # should not happen
3038
3039     push @$cpuFlags, "vendor=${cpu_vendor}"
3040         if $cpu_vendor ne 'default';
3041
3042     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3043
3044     push @$cmd, '-cpu', $cpu;
3045
3046     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3047
3048     push @$cmd, '-S' if $conf->{freeze};
3049
3050     # set keyboard layout
3051     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3052     push @$cmd, '-k', $kb if $kb;
3053
3054     # enable sound
3055     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3056     #push @$cmd, '-soundhw', 'es1370';
3057     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3058
3059     if($conf->{agent}) {
3060         my $qgasocket = qmp_socket($vmid, 1);
3061         my $pciaddr = print_pci_addr("qga0", $bridges);
3062         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3063         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3064         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3065     }
3066
3067     my $spice_port;
3068
3069     if ($qxlnum) {
3070         if ($qxlnum > 1) {
3071             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3072                 for(my $i = 1; $i < $qxlnum; $i++){
3073                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3074                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3075                 }
3076             } else {
3077                 # assume other OS works like Linux
3078                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3079                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3080             }
3081         }
3082
3083         my $pciaddr = print_pci_addr("spice", $bridges);
3084
3085         my $nodename = PVE::INotify::nodename();
3086         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3087         $spice_port = PVE::Tools::next_spice_port($pfamily);
3088
3089         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3090
3091         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3092         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3093         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3094     }
3095
3096     # enable balloon by default, unless explicitly disabled
3097     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3098         $pciaddr = print_pci_addr("balloon0", $bridges);
3099         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3100     }
3101
3102     if ($conf->{watchdog}) {
3103         my $wdopts = parse_watchdog($conf->{watchdog});
3104         $pciaddr = print_pci_addr("watchdog", $bridges);
3105         my $watchdog = $wdopts->{model} || 'i6300esb';
3106         push @$devices, '-device', "$watchdog$pciaddr";
3107         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3108     }
3109
3110     my $vollist = [];
3111     my $scsicontroller = {};
3112     my $ahcicontroller = {};
3113     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3114
3115     # Add iscsi initiator name if available
3116     if (my $initiator = get_initiator_name()) {
3117         push @$devices, '-iscsi', "initiator-name=$initiator";
3118     }
3119
3120     foreach_drive($conf, sub {
3121         my ($ds, $drive) = @_;
3122
3123         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3124             push @$vollist, $drive->{file};
3125         }
3126
3127         $use_virtio = 1 if $ds =~ m/^virtio/;
3128
3129         if (drive_is_cdrom ($drive)) {
3130             if ($bootindex_hash->{d}) {
3131                 $drive->{bootindex} = $bootindex_hash->{d};
3132                 $bootindex_hash->{d} += 1;
3133             }
3134         } else {
3135             if ($bootindex_hash->{c}) {
3136                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3137                 $bootindex_hash->{c} += 1;
3138             }
3139         }
3140
3141         if($drive->{interface} eq 'virtio'){
3142            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3143         }
3144
3145         if ($drive->{interface} eq 'scsi') {
3146
3147             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3148
3149             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3150             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3151
3152             my $iothread = '';
3153             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3154                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3155                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3156             } elsif ($drive->{iothread}) {
3157                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3158             }
3159
3160             my $queues = '';
3161             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3162                 $queues = ",num_queues=$drive->{queues}";
3163             }
3164
3165             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3166             $scsicontroller->{$controller}=1;
3167         }
3168
3169         if ($drive->{interface} eq 'sata') {
3170            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3171            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3172            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3173            $ahcicontroller->{$controller}=1;
3174         }
3175
3176         if ($drive->{interface} eq 'efidisk') {
3177             # this will be added somewhere else
3178             return;
3179         }
3180
3181         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3182         push @$devices, '-drive',$drive_cmd;
3183         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3184     });
3185
3186     for (my $i = 0; $i < $MAX_NETS; $i++) {
3187          next if !$conf->{"net$i"};
3188          my $d = parse_net($conf->{"net$i"});
3189          next if !$d;
3190
3191          $use_virtio = 1 if $d->{model} eq 'virtio';
3192
3193          if ($bootindex_hash->{n}) {
3194             $d->{bootindex} = $bootindex_hash->{n};
3195             $bootindex_hash->{n} += 1;
3196          }
3197
3198          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3199          push @$devices, '-netdev', $netdevfull;
3200
3201          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3202          push @$devices, '-device', $netdevicefull;
3203     }
3204
3205     if (!$q35) {
3206         # add pci bridges
3207         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3208            $bridges->{1} = 1;
3209            $bridges->{2} = 1;
3210         }
3211
3212         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3213
3214         while (my ($k, $v) = each %$bridges) {
3215             $pciaddr = print_pci_addr("pci.$k");
3216             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3217         }
3218     }
3219
3220     # add custom args
3221     if ($conf->{args}) {
3222         my $aa = PVE::Tools::split_args($conf->{args});
3223         push @$cmd, @$aa;
3224     }
3225
3226     push @$cmd, @$devices;
3227     push @$cmd, '-rtc', join(',', @$rtcFlags)
3228         if scalar(@$rtcFlags);
3229     push @$cmd, '-machine', join(',', @$machineFlags)
3230         if scalar(@$machineFlags);
3231     push @$cmd, '-global', join(',', @$globalFlags)
3232         if scalar(@$globalFlags);
3233
3234     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3235 }
3236
3237 sub vnc_socket {
3238     my ($vmid) = @_;
3239     return "${var_run_tmpdir}/$vmid.vnc";
3240 }
3241
3242 sub spice_port {
3243     my ($vmid) = @_;
3244
3245     my $res = vm_mon_cmd($vmid, 'query-spice');
3246
3247     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3248 }
3249
3250 sub qmp_socket {
3251     my ($vmid, $qga) = @_;
3252     my $sockettype = $qga ? 'qga' : 'qmp';
3253     return "${var_run_tmpdir}/$vmid.$sockettype";
3254 }
3255
3256 sub pidfile_name {
3257     my ($vmid) = @_;
3258     return "${var_run_tmpdir}/$vmid.pid";
3259 }
3260
3261 sub vm_devices_list {
3262     my ($vmid) = @_;
3263
3264     my $res = vm_mon_cmd($vmid, 'query-pci');
3265     my $devices = {};
3266     foreach my $pcibus (@$res) {
3267         foreach my $device (@{$pcibus->{devices}}) {
3268             next if !$device->{'qdev_id'};
3269             if ($device->{'pci_bridge'}) {
3270                 $devices->{$device->{'qdev_id'}} = 1;
3271                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3272                     next if !$bridge_device->{'qdev_id'};
3273                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3274                     $devices->{$device->{'qdev_id'}}++;
3275                 }
3276             } else {
3277                 $devices->{$device->{'qdev_id'}} = 1;
3278             }
3279         }
3280     }
3281
3282     my $resblock = vm_mon_cmd($vmid, 'query-block');
3283     foreach my $block (@$resblock) {
3284         if($block->{device} =~ m/^drive-(\S+)/){
3285                 $devices->{$1} = 1;
3286         }
3287     }
3288
3289     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3290     foreach my $mice (@$resmice) {
3291         if ($mice->{name} eq 'QEMU HID Tablet') {
3292             $devices->{tablet} = 1;
3293             last;
3294         }
3295     }
3296
3297     # for usb devices there is no query-usb
3298     # but we can iterate over the entries in
3299     # qom-list path=/machine/peripheral
3300     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3301     foreach my $per (@$resperipheral) {
3302         if ($per->{name} =~ m/^usb\d+$/) {
3303             $devices->{$per->{name}} = 1;
3304         }
3305     }
3306
3307     return $devices;
3308 }
3309
3310 sub vm_deviceplug {
3311     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3312
3313     my $q35 = machine_type_is_q35($conf);
3314
3315     my $devices_list = vm_devices_list($vmid);
3316     return 1 if defined($devices_list->{$deviceid});
3317
3318     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3319
3320     if ($deviceid eq 'tablet') {
3321
3322         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3323
3324     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3325
3326         die "usb hotplug currently not reliable\n";
3327         # since we can't reliably hot unplug all added usb devices
3328         # and usb passthrough disables live migration
3329         # we disable usb hotplugging for now
3330         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3331
3332     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3333
3334         qemu_iothread_add($vmid, $deviceid, $device);
3335
3336         qemu_driveadd($storecfg, $vmid, $device);
3337         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3338
3339         qemu_deviceadd($vmid, $devicefull);
3340         eval { qemu_deviceaddverify($vmid, $deviceid); };
3341         if (my $err = $@) {
3342             eval { qemu_drivedel($vmid, $deviceid); };
3343             warn $@ if $@;
3344             die $err;
3345         }
3346
3347     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3348
3349
3350         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3351         my $pciaddr = print_pci_addr($deviceid);
3352         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3353
3354         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3355
3356         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3357             qemu_iothread_add($vmid, $deviceid, $device);
3358             $devicefull .= ",iothread=iothread-$deviceid";
3359         }
3360
3361         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3362             $devicefull .= ",num_queues=$device->{queues}";
3363         }
3364
3365         qemu_deviceadd($vmid, $devicefull);
3366         qemu_deviceaddverify($vmid, $deviceid);
3367
3368     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3369
3370         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3371         qemu_driveadd($storecfg, $vmid, $device);
3372
3373         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3374         eval { qemu_deviceadd($vmid, $devicefull); };
3375         if (my $err = $@) {
3376             eval { qemu_drivedel($vmid, $deviceid); };
3377             warn $@ if $@;
3378             die $err;
3379         }
3380
3381     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3382
3383         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3384
3385         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3386         my $use_old_bios_files = undef;
3387         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3388
3389         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3390         qemu_deviceadd($vmid, $netdevicefull);
3391         eval { qemu_deviceaddverify($vmid, $deviceid); };
3392         if (my $err = $@) {
3393             eval { qemu_netdevdel($vmid, $deviceid); };
3394             warn $@ if $@;
3395             die $err;
3396         }
3397
3398     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3399
3400         my $bridgeid = $2;
3401         my $pciaddr = print_pci_addr($deviceid);
3402         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3403
3404         qemu_deviceadd($vmid, $devicefull);
3405         qemu_deviceaddverify($vmid, $deviceid);
3406
3407     } else {
3408         die "can't hotplug device '$deviceid'\n";
3409     }
3410
3411     return 1;
3412 }
3413
3414 # fixme: this should raise exceptions on error!
3415 sub vm_deviceunplug {
3416     my ($vmid, $conf, $deviceid) = @_;
3417
3418     my $devices_list = vm_devices_list($vmid);
3419     return 1 if !defined($devices_list->{$deviceid});
3420
3421     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3422
3423     if ($deviceid eq 'tablet') {
3424
3425         qemu_devicedel($vmid, $deviceid);
3426
3427     } elsif ($deviceid =~ m/^usb\d+$/) {
3428
3429         die "usb hotplug currently not reliable\n";
3430         # when unplugging usb devices this way,
3431         # there may be remaining usb controllers/hubs
3432         # so we disable it for now
3433         qemu_devicedel($vmid, $deviceid);
3434         qemu_devicedelverify($vmid, $deviceid);
3435
3436     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3437
3438         qemu_devicedel($vmid, $deviceid);
3439         qemu_devicedelverify($vmid, $deviceid);
3440         qemu_drivedel($vmid, $deviceid);
3441         qemu_iothread_del($conf, $vmid, $deviceid);
3442
3443     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3444
3445         qemu_devicedel($vmid, $deviceid);
3446         qemu_devicedelverify($vmid, $deviceid);
3447         qemu_iothread_del($conf, $vmid, $deviceid);
3448
3449     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3450
3451         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3452         my $device = parse_drive($deviceid, $conf->{$deviceid});
3453         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3454
3455         qemu_devicedel($vmid, $deviceid);
3456         qemu_drivedel($vmid, $deviceid);
3457         qemu_deletescsihw($conf, $vmid, $deviceid);
3458
3459     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3460
3461         qemu_devicedel($vmid, $deviceid);
3462         qemu_devicedelverify($vmid, $deviceid);
3463         qemu_netdevdel($vmid, $deviceid);
3464
3465     } else {
3466         die "can't unplug device '$deviceid'\n";
3467     }
3468
3469     return 1;
3470 }
3471
3472 sub qemu_deviceadd {
3473     my ($vmid, $devicefull) = @_;
3474
3475     $devicefull = "driver=".$devicefull;
3476     my %options =  split(/[=,]/, $devicefull);
3477
3478     vm_mon_cmd($vmid, "device_add" , %options);
3479 }
3480
3481 sub qemu_devicedel {
3482     my ($vmid, $deviceid) = @_;
3483
3484     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3485 }
3486
3487 sub qemu_iothread_add {
3488     my($vmid, $deviceid, $device) = @_;
3489
3490     if ($device->{iothread}) {
3491         my $iothreads = vm_iothreads_list($vmid);
3492         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3493     }
3494 }
3495
3496 sub qemu_iothread_del {
3497     my($conf, $vmid, $deviceid) = @_;
3498
3499     my $device = parse_drive($deviceid, $conf->{$deviceid});
3500     if ($device->{iothread}) {
3501         my $iothreads = vm_iothreads_list($vmid);
3502         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3503     }
3504 }
3505
3506 sub qemu_objectadd {
3507     my($vmid, $objectid, $qomtype) = @_;
3508
3509     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3510
3511     return 1;
3512 }
3513
3514 sub qemu_objectdel {
3515     my($vmid, $objectid) = @_;
3516
3517     vm_mon_cmd($vmid, "object-del", id => $objectid);
3518
3519     return 1;
3520 }
3521
3522 sub qemu_driveadd {
3523     my ($storecfg, $vmid, $device) = @_;
3524
3525     my $drive = print_drive_full($storecfg, $vmid, $device);
3526     $drive =~ s/\\/\\\\/g;
3527     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3528
3529     # If the command succeeds qemu prints: "OK"
3530     return 1 if $ret =~ m/OK/s;
3531
3532     die "adding drive failed: $ret\n";
3533 }
3534
3535 sub qemu_drivedel {
3536     my($vmid, $deviceid) = @_;
3537
3538     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3539     $ret =~ s/^\s+//;
3540
3541     return 1 if $ret eq "";
3542
3543     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3544     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3545
3546     die "deleting drive $deviceid failed : $ret\n";
3547 }
3548
3549 sub qemu_deviceaddverify {
3550     my ($vmid, $deviceid) = @_;
3551
3552     for (my $i = 0; $i <= 5; $i++) {
3553          my $devices_list = vm_devices_list($vmid);
3554          return 1 if defined($devices_list->{$deviceid});
3555          sleep 1;
3556     }
3557
3558     die "error on hotplug device '$deviceid'\n";
3559 }
3560
3561
3562 sub qemu_devicedelverify {
3563     my ($vmid, $deviceid) = @_;
3564
3565     # need to verify that the device is correctly removed as device_del
3566     # is async and empty return is not reliable
3567
3568     for (my $i = 0; $i <= 5; $i++) {
3569          my $devices_list = vm_devices_list($vmid);
3570          return 1 if !defined($devices_list->{$deviceid});
3571          sleep 1;
3572     }
3573
3574     die "error on hot-unplugging device '$deviceid'\n";
3575 }
3576
3577 sub qemu_findorcreatescsihw {
3578     my ($storecfg, $conf, $vmid, $device) = @_;
3579
3580     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3581
3582     my $scsihwid="$controller_prefix$controller";
3583     my $devices_list = vm_devices_list($vmid);
3584
3585     if(!defined($devices_list->{$scsihwid})) {
3586         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3587     }
3588
3589     return 1;
3590 }
3591
3592 sub qemu_deletescsihw {
3593     my ($conf, $vmid, $opt) = @_;
3594
3595     my $device = parse_drive($opt, $conf->{$opt});
3596
3597     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3598         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3599         return 1;
3600     }
3601
3602     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3603
3604     my $devices_list = vm_devices_list($vmid);
3605     foreach my $opt (keys %{$devices_list}) {
3606         if (PVE::QemuServer::is_valid_drivename($opt)) {
3607             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3608             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3609                 return 1;
3610             }
3611         }
3612     }
3613
3614     my $scsihwid="scsihw$controller";
3615
3616     vm_deviceunplug($vmid, $conf, $scsihwid);
3617
3618     return 1;
3619 }
3620
3621 sub qemu_add_pci_bridge {
3622     my ($storecfg, $conf, $vmid, $device) = @_;
3623
3624     my $bridges = {};
3625
3626     my $bridgeid;
3627
3628     print_pci_addr($device, $bridges);
3629
3630     while (my ($k, $v) = each %$bridges) {
3631         $bridgeid = $k;
3632     }
3633     return 1 if !defined($bridgeid) || $bridgeid < 1;
3634
3635     my $bridge = "pci.$bridgeid";
3636     my $devices_list = vm_devices_list($vmid);
3637
3638     if (!defined($devices_list->{$bridge})) {
3639         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3640     }
3641
3642     return 1;
3643 }
3644
3645 sub qemu_set_link_status {
3646     my ($vmid, $device, $up) = @_;
3647
3648     vm_mon_cmd($vmid, "set_link", name => $device,
3649                up => $up ? JSON::true : JSON::false);
3650 }
3651
3652 sub qemu_netdevadd {
3653     my ($vmid, $conf, $device, $deviceid) = @_;
3654
3655     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3656     my %options =  split(/[=,]/, $netdev);
3657
3658     vm_mon_cmd($vmid, "netdev_add",  %options);
3659     return 1;
3660 }
3661
3662 sub qemu_netdevdel {
3663     my ($vmid, $deviceid) = @_;
3664
3665     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3666 }
3667
3668 sub qemu_usb_hotplug {
3669     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3670
3671     return if !$device;
3672
3673     # remove the old one first
3674     vm_deviceunplug($vmid, $conf, $deviceid);
3675
3676     # check if xhci controller is necessary and available
3677     if ($device->{usb3}) {
3678
3679         my $devicelist = vm_devices_list($vmid);
3680
3681         if (!$devicelist->{xhci}) {
3682             my $pciaddr = print_pci_addr("xhci");
3683             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3684         }
3685     }
3686     my $d = parse_usb_device($device->{host});
3687     $d->{usb3} = $device->{usb3};
3688
3689     # add the new one
3690     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3691 }
3692
3693 sub qemu_cpu_hotplug {
3694     my ($vmid, $conf, $vcpus) = @_;
3695
3696     my $sockets = 1;
3697     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3698     $sockets = $conf->{sockets} if  $conf->{sockets};
3699     my $cores = $conf->{cores} || 1;
3700     my $maxcpus = $sockets * $cores;
3701
3702     $vcpus = $maxcpus if !$vcpus;
3703
3704     die "you can't add more vcpus than maxcpus\n"
3705         if $vcpus > $maxcpus;
3706
3707     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3708     die "online cpu unplug is not yet possible\n"
3709         if $vcpus < $currentvcpus;
3710
3711     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3712     die "vcpus in running vm is different than configuration\n"
3713         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3714
3715     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3716         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3717     }
3718 }
3719
3720 sub qemu_block_set_io_throttle {
3721     my ($vmid, $deviceid,
3722         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3723         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3724
3725     return if !check_running($vmid) ;
3726
3727     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3728         bps => int($bps),
3729         bps_rd => int($bps_rd),
3730         bps_wr => int($bps_wr),
3731         iops => int($iops),
3732         iops_rd => int($iops_rd),
3733         iops_wr => int($iops_wr),
3734         bps_max => int($bps_max),
3735         bps_rd_max => int($bps_rd_max),
3736         bps_wr_max => int($bps_wr_max),
3737         iops_max => int($iops_max),
3738         iops_rd_max => int($iops_rd_max),
3739         iops_wr_max => int($iops_wr_max)
3740     );
3741
3742 }
3743
3744 # old code, only used to shutdown old VM after update
3745 sub __read_avail {
3746     my ($fh, $timeout) = @_;
3747
3748     my $sel = new IO::Select;
3749     $sel->add($fh);
3750
3751     my $res = '';
3752     my $buf;
3753
3754     my @ready;
3755     while (scalar (@ready = $sel->can_read($timeout))) {
3756         my $count;
3757         if ($count = $fh->sysread($buf, 8192)) {
3758             if ($buf =~ /^(.*)\(qemu\) $/s) {
3759                 $res .= $1;
3760                 last;
3761             } else {
3762                 $res .= $buf;
3763             }
3764         } else {
3765             if (!defined($count)) {
3766                 die "$!\n";
3767             }
3768             last;
3769         }
3770     }
3771
3772     die "monitor read timeout\n" if !scalar(@ready);
3773
3774     return $res;
3775 }
3776
3777 # old code, only used to shutdown old VM after update
3778 sub vm_monitor_command {
3779     my ($vmid, $cmdstr, $nocheck) = @_;
3780
3781     my $res;
3782
3783     eval {
3784         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3785
3786         my $sname = "${var_run_tmpdir}/$vmid.mon";
3787
3788         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3789             die "unable to connect to VM $vmid socket - $!\n";
3790
3791         my $timeout = 3;
3792
3793         # hack: migrate sometime blocks the monitor (when migrate_downtime
3794         # is set)
3795         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3796             $timeout = 60*60; # 1 hour
3797         }
3798
3799         # read banner;
3800         my $data = __read_avail($sock, $timeout);
3801
3802         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3803             die "got unexpected qemu monitor banner\n";
3804         }
3805
3806         my $sel = new IO::Select;
3807         $sel->add($sock);
3808
3809         if (!scalar(my @ready = $sel->can_write($timeout))) {
3810             die "monitor write error - timeout";
3811         }
3812
3813         my $fullcmd = "$cmdstr\r";
3814
3815         # syslog('info', "VM $vmid monitor command: $cmdstr");
3816
3817         my $b;
3818         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3819             die "monitor write error - $!";
3820         }
3821
3822         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3823
3824         $timeout = 20;
3825
3826         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3827             $timeout = 60*60; # 1 hour
3828         } elsif ($cmdstr =~ m/^(eject|change)/) {
3829             $timeout = 60; # note: cdrom mount command is slow
3830         }
3831         if ($res = __read_avail($sock, $timeout)) {
3832
3833             my @lines = split("\r?\n", $res);
3834
3835             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3836
3837             $res = join("\n", @lines);
3838             $res .= "\n";
3839         }
3840     };
3841
3842     my $err = $@;
3843
3844     if ($err) {
3845         syslog("err", "VM $vmid monitor command failed - $err");
3846         die $err;
3847     }
3848
3849     return $res;
3850 }
3851
3852 sub qemu_block_resize {
3853     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3854
3855     my $running = check_running($vmid);
3856
3857     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3858
3859     return if !$running;
3860
3861     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3862
3863 }
3864
3865 sub qemu_volume_snapshot {
3866     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3867
3868     my $running = check_running($vmid);
3869
3870     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3871         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3872     } else {
3873         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3874     }
3875 }
3876
3877 sub qemu_volume_snapshot_delete {
3878     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3879
3880     my $running = check_running($vmid);
3881
3882     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3883
3884     return if !$running;
3885
3886     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3887 }
3888
3889 sub set_migration_caps {
3890     my ($vmid) = @_;
3891
3892     my $cap_ref = [];
3893
3894     my $enabled_cap = {
3895         "auto-converge" => 1,
3896         "xbzrle" => 1,
3897         "x-rdma-pin-all" => 0,
3898         "zero-blocks" => 0,
3899         "compress" => 0
3900     };
3901
3902     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3903
3904     for my $supported_capability (@$supported_capabilities) {
3905         push @$cap_ref, {
3906             capability => $supported_capability->{capability},
3907             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3908         };
3909     }
3910
3911     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3912 }
3913
3914 my $fast_plug_option = {
3915     'lock' => 1,
3916     'name' => 1,
3917     'onboot' => 1,
3918     'shares' => 1,
3919     'startup' => 1,
3920     'description' => 1,
3921     'protection' => 1,
3922 };
3923
3924 # hotplug changes in [PENDING]
3925 # $selection hash can be used to only apply specified options, for
3926 # example: { cores => 1 } (only apply changed 'cores')
3927 # $errors ref is used to return error messages
3928 sub vmconfig_hotplug_pending {
3929     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3930
3931     my $defaults = load_defaults();
3932
3933     # commit values which do not have any impact on running VM first
3934     # Note: those option cannot raise errors, we we do not care about
3935     # $selection and always apply them.
3936
3937     my $add_error = sub {
3938         my ($opt, $msg) = @_;
3939         $errors->{$opt} = "hotplug problem - $msg";
3940     };
3941
3942     my $changes = 0;
3943     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3944         if ($fast_plug_option->{$opt}) {
3945             $conf->{$opt} = $conf->{pending}->{$opt};
3946             delete $conf->{pending}->{$opt};
3947             $changes = 1;
3948         }
3949     }
3950
3951     if ($changes) {
3952         PVE::QemuConfig->write_config($vmid, $conf);
3953         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3954     }
3955
3956     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3957
3958     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3959     while (my ($opt, $force) = each %$pending_delete_hash) {
3960         next if $selection && !$selection->{$opt};
3961         eval {
3962             if ($opt eq 'hotplug') {
3963                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3964             } elsif ($opt eq 'tablet') {
3965                 die "skip\n" if !$hotplug_features->{usb};
3966                 if ($defaults->{tablet}) {
3967                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3968                 } else {
3969                     vm_deviceunplug($vmid, $conf, $opt);
3970                 }
3971             } elsif ($opt =~ m/^usb\d+/) {
3972                 die "skip\n";
3973                 # since we cannot reliably hot unplug usb devices
3974                 # we are disabling it
3975                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
3976                 vm_deviceunplug($vmid, $conf, $opt);
3977             } elsif ($opt eq 'vcpus') {
3978                 die "skip\n" if !$hotplug_features->{cpu};
3979                 qemu_cpu_hotplug($vmid, $conf, undef);
3980             } elsif ($opt eq 'balloon') {
3981                 # enable balloon device is not hotpluggable
3982                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3983             } elsif ($fast_plug_option->{$opt}) {
3984                 # do nothing
3985             } elsif ($opt =~ m/^net(\d+)$/) {
3986                 die "skip\n" if !$hotplug_features->{network};
3987                 vm_deviceunplug($vmid, $conf, $opt);
3988             } elsif (is_valid_drivename($opt)) {
3989                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3990                 vm_deviceunplug($vmid, $conf, $opt);
3991                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3992             } elsif ($opt =~ m/^memory$/) {
3993                 die "skip\n" if !$hotplug_features->{memory};
3994                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3995             } elsif ($opt eq 'cpuunits') {
3996                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3997             } elsif ($opt eq 'cpulimit') {
3998                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3999             } else {
4000                 die "skip\n";
4001             }
4002         };
4003         if (my $err = $@) {
4004             &$add_error($opt, $err) if $err ne "skip\n";
4005         } else {
4006             # save new config if hotplug was successful
4007             delete $conf->{$opt};
4008             vmconfig_undelete_pending_option($conf, $opt);
4009             PVE::QemuConfig->write_config($vmid, $conf);
4010             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4011         }
4012     }
4013
4014     foreach my $opt (keys %{$conf->{pending}}) {
4015         next if $selection && !$selection->{$opt};
4016         my $value = $conf->{pending}->{$opt};
4017         eval {
4018             if ($opt eq 'hotplug') {
4019                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4020             } elsif ($opt eq 'tablet') {
4021                 die "skip\n" if !$hotplug_features->{usb};
4022                 if ($value == 1) {
4023                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4024                 } elsif ($value == 0) {
4025                     vm_deviceunplug($vmid, $conf, $opt);
4026                 }
4027             } elsif ($opt =~ m/^usb\d+$/) {
4028                 die "skip\n";
4029                 # since we cannot reliably hot unplug usb devices
4030                 # we are disabling it
4031                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4032                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4033                 die "skip\n" if !$d;
4034                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4035             } elsif ($opt eq 'vcpus') {
4036                 die "skip\n" if !$hotplug_features->{cpu};
4037                 qemu_cpu_hotplug($vmid, $conf, $value);
4038             } elsif ($opt eq 'balloon') {
4039                 # enable/disable balloning device is not hotpluggable
4040                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4041                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4042                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4043
4044                 # allow manual ballooning if shares is set to zero
4045                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4046                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4047                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4048                 }
4049             } elsif ($opt =~ m/^net(\d+)$/) {
4050                 # some changes can be done without hotplug
4051                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4052                                     $vmid, $opt, $value);
4053             } elsif (is_valid_drivename($opt)) {
4054                 # some changes can be done without hotplug
4055                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4056                                      $vmid, $opt, $value, 1);
4057             } elsif ($opt =~ m/^memory$/) { #dimms
4058                 die "skip\n" if !$hotplug_features->{memory};
4059                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4060             } elsif ($opt eq 'cpuunits') {
4061                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4062             } elsif ($opt eq 'cpulimit') {
4063                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4064                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4065             } else {
4066                 die "skip\n";  # skip non-hot-pluggable options
4067             }
4068         };
4069         if (my $err = $@) {
4070             &$add_error($opt, $err) if $err ne "skip\n";
4071         } else {
4072             # save new config if hotplug was successful
4073             $conf->{$opt} = $value;
4074             delete $conf->{pending}->{$opt};
4075             PVE::QemuConfig->write_config($vmid, $conf);
4076             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4077         }
4078     }
4079 }
4080
4081 sub try_deallocate_drive {
4082     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4083
4084     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4085         my $volid = $drive->{file};
4086         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4087             my $sid = PVE::Storage::parse_volume_id($volid);
4088             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4089
4090             # check if the disk is really unused
4091             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4092                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4093             PVE::Storage::vdisk_free($storecfg, $volid);
4094             return 1;
4095         } else {
4096             # If vm is not owner of this disk remove from config
4097             return 1;
4098         }
4099     }
4100
4101     return undef;
4102 }
4103
4104 sub vmconfig_delete_or_detach_drive {
4105     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4106
4107     my $drive = parse_drive($opt, $conf->{$opt});
4108
4109     my $rpcenv = PVE::RPCEnvironment::get();
4110     my $authuser = $rpcenv->get_user();
4111
4112     if ($force) {
4113         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4114         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4115     } else {
4116         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4117     }
4118 }
4119
4120 sub vmconfig_apply_pending {
4121     my ($vmid, $conf, $storecfg) = @_;
4122
4123     # cold plug
4124
4125     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4126     while (my ($opt, $force) = each %$pending_delete_hash) {
4127         die "internal error" if $opt =~ m/^unused/;
4128         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4129         if (!defined($conf->{$opt})) {
4130             vmconfig_undelete_pending_option($conf, $opt);
4131             PVE::QemuConfig->write_config($vmid, $conf);
4132         } elsif (is_valid_drivename($opt)) {
4133             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4134             vmconfig_undelete_pending_option($conf, $opt);
4135             delete $conf->{$opt};
4136             PVE::QemuConfig->write_config($vmid, $conf);
4137         } else {
4138             vmconfig_undelete_pending_option($conf, $opt);
4139             delete $conf->{$opt};
4140             PVE::QemuConfig->write_config($vmid, $conf);
4141         }
4142     }
4143
4144     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4145
4146     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4147         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4148
4149         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4150             # skip if nothing changed
4151         } elsif (is_valid_drivename($opt)) {
4152             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4153                 if defined($conf->{$opt});
4154             $conf->{$opt} = $conf->{pending}->{$opt};
4155         } else {
4156             $conf->{$opt} = $conf->{pending}->{$opt};
4157         }
4158
4159         delete $conf->{pending}->{$opt};
4160         PVE::QemuConfig->write_config($vmid, $conf);
4161     }
4162 }
4163
4164 my $safe_num_ne = sub {
4165     my ($a, $b) = @_;
4166
4167     return 0 if !defined($a) && !defined($b);
4168     return 1 if !defined($a);
4169     return 1 if !defined($b);
4170
4171     return $a != $b;
4172 };
4173
4174 my $safe_string_ne = sub {
4175     my ($a, $b) = @_;
4176
4177     return 0 if !defined($a) && !defined($b);
4178     return 1 if !defined($a);
4179     return 1 if !defined($b);
4180
4181     return $a ne $b;
4182 };
4183
4184 sub vmconfig_update_net {
4185     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4186
4187     my $newnet = parse_net($value);
4188
4189     if ($conf->{$opt}) {
4190         my $oldnet = parse_net($conf->{$opt});
4191
4192         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4193             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4194             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4195             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4196
4197             # for non online change, we try to hot-unplug
4198             die "skip\n" if !$hotplug;
4199             vm_deviceunplug($vmid, $conf, $opt);
4200         } else {
4201
4202             die "internal error" if $opt !~ m/net(\d+)/;
4203             my $iface = "tap${vmid}i$1";
4204
4205             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4206                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4207                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4208                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4209                 PVE::Network::tap_unplug($iface);
4210                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4211             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4212                 # Rate can be applied on its own but any change above needs to
4213                 # include the rate in tap_plug since OVS resets everything.
4214                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4215             }
4216
4217             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4218                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4219             }
4220
4221             return 1;
4222         }
4223     }
4224
4225     if ($hotplug) {
4226         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4227     } else {
4228         die "skip\n";
4229     }
4230 }
4231
4232 sub vmconfig_update_disk {
4233     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4234
4235     # fixme: do we need force?
4236
4237     my $drive = parse_drive($opt, $value);
4238
4239     if ($conf->{$opt}) {
4240
4241         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4242
4243             my $media = $drive->{media} || 'disk';
4244             my $oldmedia = $old_drive->{media} || 'disk';
4245             die "unable to change media type\n" if $media ne $oldmedia;
4246
4247             if (!drive_is_cdrom($old_drive)) {
4248
4249                 if ($drive->{file} ne $old_drive->{file}) {
4250
4251                     die "skip\n" if !$hotplug;
4252
4253                     # unplug and register as unused
4254                     vm_deviceunplug($vmid, $conf, $opt);
4255                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4256
4257                 } else {
4258                     # update existing disk
4259
4260                     # skip non hotpluggable value
4261                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4262                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4263                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4264                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4265                         die "skip\n";
4266                     }
4267
4268                     # apply throttle
4269                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4270                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4271                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4272                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4273                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4274                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4275                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4276                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4277                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4278                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4279                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4280                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4281
4282                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4283                                                    ($drive->{mbps} || 0)*1024*1024,
4284                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4285                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4286                                                    $drive->{iops} || 0,
4287                                                    $drive->{iops_rd} || 0,
4288                                                    $drive->{iops_wr} || 0,
4289                                                    ($drive->{mbps_max} || 0)*1024*1024,
4290                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4291                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4292                                                    $drive->{iops_max} || 0,
4293                                                    $drive->{iops_rd_max} || 0,
4294                                                    $drive->{iops_wr_max} || 0);
4295
4296                     }
4297
4298                     return 1;
4299                 }
4300
4301             } else { # cdrom
4302
4303                 if ($drive->{file} eq 'none') {
4304                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4305                 } else {
4306                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4307                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4308                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4309                 }
4310
4311                 return 1;
4312             }
4313         }
4314     }
4315
4316     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4317     # hotplug new disks
4318     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4319     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4320 }
4321
4322 sub vm_start {
4323     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4324         $forcemachine, $spice_ticket) = @_;
4325
4326     PVE::QemuConfig->lock_config($vmid, sub {
4327         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4328
4329         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4330
4331         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4332
4333         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4334
4335         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4336             vmconfig_apply_pending($vmid, $conf, $storecfg);
4337             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4338         }
4339
4340         my $defaults = load_defaults();
4341
4342         # set environment variable useful inside network script
4343         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4344
4345         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4346
4347         my $migrate_port = 0;
4348         my $migrate_uri;
4349         if ($statefile) {
4350             if ($statefile eq 'tcp') {
4351                 my $localip = "localhost";
4352                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4353                 my $nodename = PVE::INotify::nodename();
4354                 if ($datacenterconf->{migration_unsecure}) {
4355                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4356                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4357                 }
4358                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4359                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4360                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4361                 push @$cmd, '-incoming', $migrate_uri;
4362                 push @$cmd, '-S';
4363
4364             } elsif ($statefile eq 'unix') {
4365                 # should be default for secure migrations as a ssh TCP forward
4366                 # tunnel is not deterministic reliable ready and fails regurarly
4367                 # to set up in time, so use UNIX socket forwards
4368                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4369                 unlink $socket_addr;
4370
4371                 $migrate_uri = "unix:$socket_addr";
4372
4373                 push @$cmd, '-incoming', $migrate_uri;
4374                 push @$cmd, '-S';
4375
4376             } else {
4377                 push @$cmd, '-loadstate', $statefile;
4378             }
4379         } elsif ($paused) {
4380             push @$cmd, '-S';
4381         }
4382
4383         # host pci devices
4384         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4385           my $d = parse_hostpci($conf->{"hostpci$i"});
4386           next if !$d;
4387           my $pcidevices = $d->{pciid};
4388           foreach my $pcidevice (@$pcidevices) {
4389                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4390
4391                 my $info = pci_device_info("0000:$pciid");
4392                 die "IOMMU not present\n" if !check_iommu_support();
4393                 die "no pci device info for device '$pciid'\n" if !$info;
4394                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4395                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4396           }
4397         }
4398
4399         PVE::Storage::activate_volumes($storecfg, $vollist);
4400
4401         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4402             my $cmd = [];
4403             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4404             eval  { run_command($cmd); };
4405         }
4406
4407         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4408                                                   : $defaults->{cpuunits};
4409
4410         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4411
4412         my %properties = (
4413             Slice => 'qemu.slice',
4414             KillMode => 'none',
4415             CPUShares => $cpuunits
4416         );
4417
4418         if (my $cpulimit = $conf->{cpulimit}) {
4419             $properties{CPUQuota} = int($cpulimit * 100);
4420         }
4421         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4422
4423         if ($conf->{hugepages}) {
4424
4425             my $code = sub {
4426                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4427                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4428
4429                 PVE::QemuServer::Memory::hugepages_mount();
4430                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4431
4432                 eval  {
4433                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4434                     run_command($cmd, %run_params);
4435                 };
4436
4437                 if (my $err = $@) {
4438                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4439                     die $err;
4440                 }
4441
4442                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4443             };
4444             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4445
4446         } else {
4447             eval  {
4448                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4449                 run_command($cmd, %run_params);
4450             };
4451         }
4452
4453         if (my $err = $@) {
4454             # deactivate volumes if start fails
4455             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4456             die "start failed: $err";
4457         }
4458
4459         print "migration listens on $migrate_uri\n" if $migrate_uri;
4460
4461         if ($statefile && $statefile ne 'tcp')  {
4462             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4463             warn $@ if $@;
4464         }
4465
4466         if ($migratedfrom) {
4467
4468             eval {
4469                 set_migration_caps($vmid);
4470             };
4471             warn $@ if $@;
4472
4473             if ($spice_port) {
4474                 print "spice listens on port $spice_port\n";
4475                 if ($spice_ticket) {
4476                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4477                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4478                 }
4479             }
4480
4481         } else {
4482
4483             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4484                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4485                     if $conf->{balloon};
4486             }
4487
4488             foreach my $opt (keys %$conf) {
4489                 next if $opt !~  m/^net\d+$/;
4490                 my $nicconf = parse_net($conf->{$opt});
4491                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4492             }
4493         }
4494
4495         vm_mon_cmd_nocheck($vmid, 'qom-set',
4496                     path => "machine/peripheral/balloon0",
4497                     property => "guest-stats-polling-interval",
4498                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4499
4500     });
4501 }
4502
4503 sub vm_mon_cmd {
4504     my ($vmid, $execute, %params) = @_;
4505
4506     my $cmd = { execute => $execute, arguments => \%params };
4507     vm_qmp_command($vmid, $cmd);
4508 }
4509
4510 sub vm_mon_cmd_nocheck {
4511     my ($vmid, $execute, %params) = @_;
4512
4513     my $cmd = { execute => $execute, arguments => \%params };
4514     vm_qmp_command($vmid, $cmd, 1);
4515 }
4516
4517 sub vm_qmp_command {
4518     my ($vmid, $cmd, $nocheck) = @_;
4519
4520     my $res;
4521
4522     my $timeout;
4523     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4524         $timeout = $cmd->{arguments}->{timeout};
4525         delete $cmd->{arguments}->{timeout};
4526     }
4527
4528     eval {
4529         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4530         my $sname = qmp_socket($vmid);
4531         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4532             my $qmpclient = PVE::QMPClient->new();
4533
4534             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4535         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4536             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4537                 if scalar(%{$cmd->{arguments}});
4538             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4539         } else {
4540             die "unable to open monitor socket\n";
4541         }
4542     };
4543     if (my $err = $@) {
4544         syslog("err", "VM $vmid qmp command failed - $err");
4545         die $err;
4546     }
4547
4548     return $res;
4549 }
4550
4551 sub vm_human_monitor_command {
4552     my ($vmid, $cmdline) = @_;
4553
4554     my $res;
4555
4556     my $cmd = {
4557         execute => 'human-monitor-command',
4558         arguments => { 'command-line' => $cmdline},
4559     };
4560
4561     return vm_qmp_command($vmid, $cmd);
4562 }
4563
4564 sub vm_commandline {
4565     my ($storecfg, $vmid) = @_;
4566
4567     my $conf = PVE::QemuConfig->load_config($vmid);
4568
4569     my $defaults = load_defaults();
4570
4571     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4572
4573     return join(' ', @$cmd);
4574 }
4575
4576 sub vm_reset {
4577     my ($vmid, $skiplock) = @_;
4578
4579     PVE::QemuConfig->lock_config($vmid, sub {
4580
4581         my $conf = PVE::QemuConfig->load_config($vmid);
4582
4583         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4584
4585         vm_mon_cmd($vmid, "system_reset");
4586     });
4587 }
4588
4589 sub get_vm_volumes {
4590     my ($conf) = @_;
4591
4592     my $vollist = [];
4593     foreach_volid($conf, sub {
4594         my ($volid, $is_cdrom) = @_;
4595
4596         return if $volid =~ m|^/|;
4597
4598         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4599         return if !$sid;
4600
4601         push @$vollist, $volid;
4602     });
4603
4604     return $vollist;
4605 }
4606
4607 sub vm_stop_cleanup {
4608     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4609
4610     eval {
4611
4612         if (!$keepActive) {
4613             my $vollist = get_vm_volumes($conf);
4614             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4615         }
4616
4617         foreach my $ext (qw(mon qmp pid vnc qga)) {
4618             unlink "/var/run/qemu-server/${vmid}.$ext";
4619         }
4620
4621         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4622     };
4623     warn $@ if $@; # avoid errors - just warn
4624 }
4625
4626 # Note: use $nockeck to skip tests if VM configuration file exists.
4627 # We need that when migration VMs to other nodes (files already moved)
4628 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4629 sub vm_stop {
4630     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4631
4632     $force = 1 if !defined($force) && !$shutdown;
4633
4634     if ($migratedfrom){
4635         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4636         kill 15, $pid if $pid;
4637         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4638         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4639         return;
4640     }
4641
4642     PVE::QemuConfig->lock_config($vmid, sub {
4643
4644         my $pid = check_running($vmid, $nocheck);
4645         return if !$pid;
4646
4647         my $conf;
4648         if (!$nocheck) {
4649             $conf = PVE::QemuConfig->load_config($vmid);
4650             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4651             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4652                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4653                 $timeout = $opts->{down} if $opts->{down};
4654             }
4655         }
4656
4657         $timeout = 60 if !defined($timeout);
4658
4659         eval {
4660             if ($shutdown) {
4661                 if (defined($conf) && $conf->{agent}) {
4662                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4663                 } else {
4664                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4665                 }
4666             } else {
4667                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4668             }
4669         };
4670         my $err = $@;
4671
4672         if (!$err) {
4673             my $count = 0;
4674             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4675                 $count++;
4676                 sleep 1;
4677             }
4678
4679             if ($count >= $timeout) {
4680                 if ($force) {
4681                     warn "VM still running - terminating now with SIGTERM\n";
4682                     kill 15, $pid;
4683                 } else {
4684                     die "VM quit/powerdown failed - got timeout\n";
4685                 }
4686             } else {
4687                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4688                 return;
4689             }
4690         } else {
4691             if ($force) {
4692                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4693                 kill 15, $pid;
4694             } else {
4695                 die "VM quit/powerdown failed\n";
4696             }
4697         }
4698
4699         # wait again
4700         $timeout = 10;
4701
4702         my $count = 0;
4703         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4704             $count++;
4705             sleep 1;
4706         }
4707
4708         if ($count >= $timeout) {
4709             warn "VM still running - terminating now with SIGKILL\n";
4710             kill 9, $pid;
4711             sleep 1;
4712         }
4713
4714         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4715    });
4716 }
4717
4718 sub vm_suspend {
4719     my ($vmid, $skiplock) = @_;
4720
4721     PVE::QemuConfig->lock_config($vmid, sub {
4722
4723         my $conf = PVE::QemuConfig->load_config($vmid);
4724
4725         PVE::QemuConfig->check_lock($conf)
4726             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4727
4728         vm_mon_cmd($vmid, "stop");
4729     });
4730 }
4731
4732 sub vm_resume {
4733     my ($vmid, $skiplock, $nocheck) = @_;
4734
4735     PVE::QemuConfig->lock_config($vmid, sub {
4736
4737         if (!$nocheck) {
4738
4739             my $conf = PVE::QemuConfig->load_config($vmid);
4740
4741             PVE::QemuConfig->check_lock($conf)
4742                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4743
4744             vm_mon_cmd($vmid, "cont");
4745
4746         } else {
4747             vm_mon_cmd_nocheck($vmid, "cont");
4748         }
4749     });
4750 }
4751
4752 sub vm_sendkey {
4753     my ($vmid, $skiplock, $key) = @_;
4754
4755     PVE::QemuConfig->lock_config($vmid, sub {
4756
4757         my $conf = PVE::QemuConfig->load_config($vmid);
4758
4759         # there is no qmp command, so we use the human monitor command
4760         vm_human_monitor_command($vmid, "sendkey $key");
4761     });
4762 }
4763
4764 sub vm_destroy {
4765     my ($storecfg, $vmid, $skiplock) = @_;
4766
4767     PVE::QemuConfig->lock_config($vmid, sub {
4768
4769         my $conf = PVE::QemuConfig->load_config($vmid);
4770
4771         if (!check_running($vmid)) {
4772             destroy_vm($storecfg, $vmid, undef, $skiplock);
4773         } else {
4774             die "VM $vmid is running - destroy failed\n";
4775         }
4776     });
4777 }
4778
4779 # pci helpers
4780
4781 sub file_write {
4782     my ($filename, $buf) = @_;
4783
4784     my $fh = IO::File->new($filename, "w");
4785     return undef if !$fh;
4786
4787     my $res = print $fh $buf;
4788
4789     $fh->close();
4790
4791     return $res;
4792 }
4793
4794 sub pci_device_info {
4795     my ($name) = @_;
4796
4797     my $res;
4798
4799     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4800     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4801
4802     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4803     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4804
4805     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4806     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4807
4808     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4809     return undef if !defined($product) || $product !~ s/^0x//;
4810
4811     $res = {
4812         name => $name,
4813         vendor => $vendor,
4814         product => $product,
4815         domain => $domain,
4816         bus => $bus,
4817         slot => $slot,
4818         func => $func,
4819         irq => $irq,
4820         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4821     };
4822
4823     return $res;
4824 }
4825
4826 sub pci_dev_reset {
4827     my ($dev) = @_;
4828
4829     my $name = $dev->{name};
4830
4831     my $fn = "$pcisysfs/devices/$name/reset";
4832
4833     return file_write($fn, "1");
4834 }
4835
4836 sub pci_dev_bind_to_vfio {
4837     my ($dev) = @_;
4838
4839     my $name = $dev->{name};
4840
4841     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4842
4843     if (!-d $vfio_basedir) {
4844         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4845     }
4846     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4847
4848     my $testdir = "$vfio_basedir/$name";
4849     return 1 if -d $testdir;
4850
4851     my $data = "$dev->{vendor} $dev->{product}";
4852     return undef if !file_write("$vfio_basedir/new_id", $data);
4853
4854     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4855     if (!file_write($fn, $name)) {
4856         return undef if -f $fn;
4857     }
4858
4859     $fn = "$vfio_basedir/bind";
4860     if (! -d $testdir) {
4861         return undef if !file_write($fn, $name);
4862     }
4863
4864     return -d $testdir;
4865 }
4866
4867 sub pci_dev_group_bind_to_vfio {
4868     my ($pciid) = @_;
4869
4870     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4871
4872     if (!-d $vfio_basedir) {
4873         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4874     }
4875     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4876
4877     # get IOMMU group devices
4878     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4879       my @devs = grep /^0000:/, readdir($D);
4880     closedir($D);
4881
4882     foreach my $pciid (@devs) {
4883         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4884
4885         # pci bridges, switches or root ports are not supported
4886         # they have a pci_bus subdirectory so skip them
4887         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4888
4889         my $info = pci_device_info($1);
4890         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4891     }
4892
4893     return 1;
4894 }
4895
4896 # vzdump restore implementaion
4897
4898 sub tar_archive_read_firstfile {
4899     my $archive = shift;
4900
4901     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4902
4903     # try to detect archive type first
4904     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
4905         die "unable to open file '$archive'\n";
4906     my $firstfile = <$fh>;
4907     kill 15, $pid;
4908     close $fh;
4909
4910     die "ERROR: archive contaions no data\n" if !$firstfile;
4911     chomp $firstfile;
4912
4913     return $firstfile;
4914 }
4915
4916 sub tar_restore_cleanup {
4917     my ($storecfg, $statfile) = @_;
4918
4919     print STDERR "starting cleanup\n";
4920
4921     if (my $fd = IO::File->new($statfile, "r")) {
4922         while (defined(my $line = <$fd>)) {
4923             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4924                 my $volid = $2;
4925                 eval {
4926                     if ($volid =~ m|^/|) {
4927                         unlink $volid || die 'unlink failed\n';
4928                     } else {
4929                         PVE::Storage::vdisk_free($storecfg, $volid);
4930                     }
4931                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4932                 };
4933                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4934             } else {
4935                 print STDERR "unable to parse line in statfile - $line";
4936             }
4937         }
4938         $fd->close();
4939     }
4940 }
4941
4942 sub restore_archive {
4943     my ($archive, $vmid, $user, $opts) = @_;
4944
4945     my $format = $opts->{format};
4946     my $comp;
4947
4948     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4949         $format = 'tar' if !$format;
4950         $comp = 'gzip';
4951     } elsif ($archive =~ m/\.tar$/) {
4952         $format = 'tar' if !$format;
4953     } elsif ($archive =~ m/.tar.lzo$/) {
4954         $format = 'tar' if !$format;
4955         $comp = 'lzop';
4956     } elsif ($archive =~ m/\.vma$/) {
4957         $format = 'vma' if !$format;
4958     } elsif ($archive =~ m/\.vma\.gz$/) {
4959         $format = 'vma' if !$format;
4960         $comp = 'gzip';
4961     } elsif ($archive =~ m/\.vma\.lzo$/) {
4962         $format = 'vma' if !$format;
4963         $comp = 'lzop';
4964     } else {
4965         $format = 'vma' if !$format; # default
4966     }
4967
4968     # try to detect archive format
4969     if ($format eq 'tar') {
4970         return restore_tar_archive($archive, $vmid, $user, $opts);
4971     } else {
4972         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4973     }
4974 }
4975
4976 sub restore_update_config_line {
4977     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4978
4979     return if $line =~ m/^\#qmdump\#/;
4980     return if $line =~ m/^\#vzdump\#/;
4981     return if $line =~ m/^lock:/;
4982     return if $line =~ m/^unused\d+:/;
4983     return if $line =~ m/^parent:/;
4984     return if $line =~ m/^template:/; # restored VM is never a template
4985
4986     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
4987     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4988         # try to convert old 1.X settings
4989         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4990         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4991             my ($model, $macaddr) = split(/\=/, $devconfig);
4992             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
4993             my $net = {
4994                 model => $model,
4995                 bridge => "vmbr$ind",
4996                 macaddr => $macaddr,
4997             };
4998             my $netstr = print_net($net);
4999
5000             print $outfd "net$cookie->{netcount}: $netstr\n";
5001             $cookie->{netcount}++;
5002         }
5003     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5004         my ($id, $netstr) = ($1, $2);
5005         my $net = parse_net($netstr);
5006         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5007         $netstr = print_net($net);
5008         print $outfd "$id: $netstr\n";
5009     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5010         my $virtdev = $1;
5011         my $value = $3;
5012         my $di = parse_drive($virtdev, $value);
5013         if (defined($di->{backup}) && !$di->{backup}) {
5014             print $outfd "#$line";
5015         } elsif ($map->{$virtdev}) {
5016             delete $di->{format}; # format can change on restore
5017             $di->{file} = $map->{$virtdev};
5018             $value = print_drive($vmid, $di);
5019             print $outfd "$virtdev: $value\n";
5020         } else {
5021             print $outfd $line;
5022         }
5023     } else {
5024         print $outfd $line;
5025     }
5026 }
5027
5028 sub scan_volids {
5029     my ($cfg, $vmid) = @_;
5030
5031     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5032
5033     my $volid_hash = {};
5034     foreach my $storeid (keys %$info) {
5035         foreach my $item (@{$info->{$storeid}}) {
5036             next if !($item->{volid} && $item->{size});
5037             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5038             $volid_hash->{$item->{volid}} = $item;
5039         }
5040     }
5041
5042     return $volid_hash;
5043 }
5044
5045 sub is_volume_in_use {
5046     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5047
5048     my $path = PVE::Storage::path($storecfg, $volid);
5049
5050     my $scan_config = sub {
5051         my ($cref, $snapname) = @_;
5052
5053         foreach my $key (keys %$cref) {
5054             my $value = $cref->{$key};
5055             if (is_valid_drivename($key)) {
5056                 next if $skip_drive && $key eq $skip_drive;
5057                 my $drive = parse_drive($key, $value);
5058                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5059                 return 1 if $volid eq $drive->{file};
5060                 if ($drive->{file} =~ m!^/!) {
5061                     return 1 if $drive->{file} eq $path;
5062                 } else {
5063                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5064                     next if !$storeid;
5065                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5066                     next if !$scfg;
5067                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5068                 }
5069             }
5070         }
5071
5072         return 0;
5073     };
5074
5075     return 1 if &$scan_config($conf);
5076
5077     undef $skip_drive;
5078
5079     foreach my $snapname (keys %{$conf->{snapshots}}) {
5080         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5081     }
5082
5083     return 0;
5084 }
5085
5086 sub update_disksize {
5087     my ($vmid, $conf, $volid_hash) = @_;
5088
5089     my $changes;
5090
5091     my $used = {};
5092
5093     # Note: it is allowed to define multiple storages with same path (alias), so
5094     # we need to check both 'volid' and real 'path' (two different volid can point
5095     # to the same path).
5096
5097     my $usedpath = {};
5098
5099     # update size info
5100     foreach my $opt (keys %$conf) {
5101         if (is_valid_drivename($opt)) {
5102             my $drive = parse_drive($opt, $conf->{$opt});
5103             my $volid = $drive->{file};
5104             next if !$volid;
5105
5106             $used->{$volid} = 1;
5107             if ($volid_hash->{$volid} &&
5108                 (my $path = $volid_hash->{$volid}->{path})) {
5109                 $usedpath->{$path} = 1;
5110             }
5111
5112             next if drive_is_cdrom($drive);
5113             next if !$volid_hash->{$volid};
5114
5115             $drive->{size} = $volid_hash->{$volid}->{size};
5116             my $new = print_drive($vmid, $drive);
5117             if ($new ne $conf->{$opt}) {
5118                 $changes = 1;
5119                 $conf->{$opt} = $new;
5120             }
5121         }
5122     }
5123
5124     # remove 'unusedX' entry if volume is used
5125     foreach my $opt (keys %$conf) {
5126         next if $opt !~ m/^unused\d+$/;
5127         my $volid = $conf->{$opt};
5128         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5129         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5130             $changes = 1;
5131             delete $conf->{$opt};
5132         }
5133     }
5134
5135     foreach my $volid (sort keys %$volid_hash) {
5136         next if $volid =~ m/vm-$vmid-state-/;
5137         next if $used->{$volid};
5138         my $path = $volid_hash->{$volid}->{path};
5139         next if !$path; # just to be sure
5140         next if $usedpath->{$path};
5141         $changes = 1;
5142         PVE::QemuConfig->add_unused_volume($conf, $volid);
5143         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5144     }
5145
5146     return $changes;
5147 }
5148
5149 sub rescan {
5150     my ($vmid, $nolock) = @_;
5151
5152     my $cfg = PVE::Storage::config();
5153
5154     my $volid_hash = scan_volids($cfg, $vmid);
5155
5156     my $updatefn =  sub {
5157         my ($vmid) = @_;
5158
5159         my $conf = PVE::QemuConfig->load_config($vmid);
5160
5161         PVE::QemuConfig->check_lock($conf);
5162
5163         my $vm_volids = {};
5164         foreach my $volid (keys %$volid_hash) {
5165             my $info = $volid_hash->{$volid};
5166             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5167         }
5168
5169         my $changes = update_disksize($vmid, $conf, $vm_volids);
5170
5171         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5172     };
5173
5174     if (defined($vmid)) {
5175         if ($nolock) {
5176             &$updatefn($vmid);
5177         } else {
5178             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5179         }
5180     } else {
5181         my $vmlist = config_list();
5182         foreach my $vmid (keys %$vmlist) {
5183             if ($nolock) {
5184                 &$updatefn($vmid);
5185             } else {
5186                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5187             }
5188         }
5189     }
5190 }
5191
5192 sub restore_vma_archive {
5193     my ($archive, $vmid, $user, $opts, $comp) = @_;
5194
5195     my $input = $archive eq '-' ? "<&STDIN" : undef;
5196     my $readfrom = $archive;
5197
5198     my $uncomp = '';
5199     if ($comp) {
5200         $readfrom = '-';
5201         my $qarchive = PVE::Tools::shellquote($archive);
5202         if ($comp eq 'gzip') {
5203             $uncomp = "zcat $qarchive|";
5204         } elsif ($comp eq 'lzop') {
5205             $uncomp = "lzop -d -c $qarchive|";
5206         } else {
5207             die "unknown compression method '$comp'\n";
5208         }
5209
5210     }
5211
5212     my $tmpdir = "/var/tmp/vzdumptmp$$";
5213     rmtree $tmpdir;
5214
5215     # disable interrupts (always do cleanups)
5216     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5217         warn "got interrupt - ignored\n";
5218     };
5219
5220     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5221     POSIX::mkfifo($mapfifo, 0600);
5222     my $fifofh;
5223
5224     my $openfifo = sub {
5225         open($fifofh, '>', $mapfifo) || die $!;
5226     };
5227
5228     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5229
5230     my $oldtimeout;
5231     my $timeout = 5;
5232
5233     my $devinfo = {};
5234
5235     my $rpcenv = PVE::RPCEnvironment::get();
5236
5237     my $conffile = PVE::QemuConfig->config_file($vmid);
5238     my $tmpfn = "$conffile.$$.tmp";
5239
5240     # Note: $oldconf is undef if VM does not exists
5241     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5242     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5243
5244     my $print_devmap = sub {
5245         my $virtdev_hash = {};
5246
5247         my $cfgfn = "$tmpdir/qemu-server.conf";
5248
5249         # we can read the config - that is already extracted
5250         my $fh = IO::File->new($cfgfn, "r") ||
5251             "unable to read qemu-server.conf - $!\n";
5252
5253         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5254         if (-f $fwcfgfn) {
5255             my $pve_firewall_dir = '/etc/pve/firewall';
5256             mkdir $pve_firewall_dir; # make sure the dir exists
5257             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5258         }
5259
5260         while (defined(my $line = <$fh>)) {
5261             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5262                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5263                 die "archive does not contain data for drive '$virtdev'\n"
5264                     if !$devinfo->{$devname};
5265                 if (defined($opts->{storage})) {
5266                     $storeid = $opts->{storage} || 'local';
5267                 } elsif (!$storeid) {
5268                     $storeid = 'local';
5269                 }
5270                 $format = 'raw' if !$format;
5271                 $devinfo->{$devname}->{devname} = $devname;
5272                 $devinfo->{$devname}->{virtdev} = $virtdev;
5273                 $devinfo->{$devname}->{format} = $format;
5274                 $devinfo->{$devname}->{storeid} = $storeid;
5275
5276                 # check permission on storage
5277                 my $pool = $opts->{pool}; # todo: do we need that?
5278                 if ($user ne 'root@pam') {
5279                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5280                 }
5281
5282                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5283             }
5284         }
5285
5286         foreach my $devname (keys %$devinfo) {
5287             die "found no device mapping information for device '$devname'\n"
5288                 if !$devinfo->{$devname}->{virtdev};
5289         }
5290
5291         my $cfg = PVE::Storage::config();
5292
5293         # create empty/temp config
5294         if ($oldconf) {
5295             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5296             foreach_drive($oldconf, sub {
5297                 my ($ds, $drive) = @_;
5298
5299                 return if drive_is_cdrom($drive);
5300
5301                 my $volid = $drive->{file};
5302
5303                 return if !$volid || $volid =~ m|^/|;
5304
5305                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5306                 return if !$path || !$owner || ($owner != $vmid);
5307
5308                 # Note: only delete disk we want to restore
5309                 # other volumes will become unused
5310                 if ($virtdev_hash->{$ds}) {
5311                     PVE::Storage::vdisk_free($cfg, $volid);
5312                 }
5313             });
5314
5315             # delete vmstate files
5316             # since after the restore we have no snapshots anymore
5317             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5318                 my $snap = $oldconf->{snapshots}->{$snapname};
5319                 if ($snap->{vmstate}) {
5320                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5321                     if (my $err = $@) {
5322                         warn $err;
5323                     }
5324                 }
5325             }
5326         }
5327
5328         my $map = {};
5329         foreach my $virtdev (sort keys %$virtdev_hash) {
5330             my $d = $virtdev_hash->{$virtdev};
5331             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5332             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5333
5334             # test if requested format is supported
5335             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5336             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5337             $d->{format} = $defFormat if !$supported;
5338
5339             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5340                                                   $d->{format}, undef, $alloc_size);
5341             print STDERR "new volume ID is '$volid'\n";
5342             $d->{volid} = $volid;
5343             my $path = PVE::Storage::path($cfg, $volid);
5344
5345             PVE::Storage::activate_volumes($cfg,[$volid]);
5346
5347             my $write_zeros = 1;
5348             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5349                 $write_zeros = 0;
5350             }
5351
5352             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5353
5354             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5355             $map->{$virtdev} = $volid;
5356         }
5357
5358         $fh->seek(0, 0) || die "seek failed - $!\n";
5359
5360         my $outfd = new IO::File ($tmpfn, "w") ||
5361             die "unable to write config for VM $vmid\n";
5362
5363         my $cookie = { netcount => 0 };
5364         while (defined(my $line = <$fh>)) {
5365             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5366         }
5367
5368         $fh->close();
5369         $outfd->close();
5370     };
5371
5372     eval {
5373         # enable interrupts
5374         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5375             die "interrupted by signal\n";
5376         };
5377         local $SIG{ALRM} = sub { die "got timeout\n"; };
5378
5379         $oldtimeout = alarm($timeout);
5380
5381         my $parser = sub {
5382             my $line = shift;
5383
5384             print "$line\n";
5385
5386             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5387                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5388                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5389             } elsif ($line =~ m/^CTIME: /) {
5390                 # we correctly received the vma config, so we can disable
5391                 # the timeout now for disk allocation (set to 10 minutes, so
5392                 # that we always timeout if something goes wrong)
5393                 alarm(600);
5394                 &$print_devmap();
5395                 print $fifofh "done\n";
5396                 my $tmp = $oldtimeout || 0;
5397                 $oldtimeout = undef;
5398                 alarm($tmp);
5399                 close($fifofh);
5400             }
5401         };
5402
5403         print "restore vma archive: $cmd\n";
5404         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5405     };
5406     my $err = $@;
5407
5408     alarm($oldtimeout) if $oldtimeout;
5409
5410     my $vollist = [];
5411     foreach my $devname (keys %$devinfo) {
5412         my $volid = $devinfo->{$devname}->{volid};
5413         push @$vollist, $volid if $volid;
5414     }
5415
5416     my $cfg = PVE::Storage::config();
5417     PVE::Storage::deactivate_volumes($cfg, $vollist);
5418
5419     unlink $mapfifo;
5420
5421     if ($err) {
5422         rmtree $tmpdir;
5423         unlink $tmpfn;
5424
5425         foreach my $devname (keys %$devinfo) {
5426             my $volid = $devinfo->{$devname}->{volid};
5427             next if !$volid;
5428             eval {
5429                 if ($volid =~ m|^/|) {
5430                     unlink $volid || die 'unlink failed\n';
5431                 } else {
5432                     PVE::Storage::vdisk_free($cfg, $volid);
5433                 }
5434                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5435             };
5436             print STDERR "unable to cleanup '$volid' - $@" if $@;
5437         }
5438         die $err;
5439     }
5440
5441     rmtree $tmpdir;
5442
5443     rename($tmpfn, $conffile) ||
5444         die "unable to commit configuration file '$conffile'\n";
5445
5446     PVE::Cluster::cfs_update(); # make sure we read new file
5447
5448     eval { rescan($vmid, 1); };
5449     warn $@ if $@;
5450 }
5451
5452 sub restore_tar_archive {
5453     my ($archive, $vmid, $user, $opts) = @_;
5454
5455     if ($archive ne '-') {
5456         my $firstfile = tar_archive_read_firstfile($archive);
5457         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5458             if $firstfile ne 'qemu-server.conf';
5459     }
5460
5461     my $storecfg = PVE::Storage::config();
5462
5463     # destroy existing data - keep empty config
5464     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5465     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5466
5467     my $tocmd = "/usr/lib/qemu-server/qmextract";
5468
5469     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5470     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5471     $tocmd .= ' --prealloc' if $opts->{prealloc};
5472     $tocmd .= ' --info' if $opts->{info};
5473
5474     # tar option "xf" does not autodetect compression when read from STDIN,
5475     # so we pipe to zcat
5476     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5477         PVE::Tools::shellquote("--to-command=$tocmd");
5478
5479     my $tmpdir = "/var/tmp/vzdumptmp$$";
5480     mkpath $tmpdir;
5481
5482     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5483     local $ENV{VZDUMP_VMID} = $vmid;
5484     local $ENV{VZDUMP_USER} = $user;
5485
5486     my $conffile = PVE::QemuConfig->config_file($vmid);
5487     my $tmpfn = "$conffile.$$.tmp";
5488
5489     # disable interrupts (always do cleanups)
5490     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5491         print STDERR "got interrupt - ignored\n";
5492     };
5493
5494     eval {
5495         # enable interrupts
5496         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5497             die "interrupted by signal\n";
5498         };
5499
5500         if ($archive eq '-') {
5501             print "extracting archive from STDIN\n";
5502             run_command($cmd, input => "<&STDIN");
5503         } else {
5504             print "extracting archive '$archive'\n";
5505             run_command($cmd);
5506         }
5507
5508         return if $opts->{info};
5509
5510         # read new mapping
5511         my $map = {};
5512         my $statfile = "$tmpdir/qmrestore.stat";
5513         if (my $fd = IO::File->new($statfile, "r")) {
5514             while (defined (my $line = <$fd>)) {
5515                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5516                     $map->{$1} = $2 if $1;
5517                 } else {
5518                     print STDERR "unable to parse line in statfile - $line\n";
5519                 }
5520             }
5521             $fd->close();
5522         }
5523
5524         my $confsrc = "$tmpdir/qemu-server.conf";
5525
5526         my $srcfd = new IO::File($confsrc, "r") ||
5527             die "unable to open file '$confsrc'\n";
5528
5529         my $outfd = new IO::File ($tmpfn, "w") ||
5530             die "unable to write config for VM $vmid\n";
5531
5532         my $cookie = { netcount => 0 };
5533         while (defined (my $line = <$srcfd>)) {
5534             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5535         }
5536
5537         $srcfd->close();
5538         $outfd->close();
5539     };
5540     my $err = $@;
5541
5542     if ($err) {
5543
5544         unlink $tmpfn;
5545
5546         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5547
5548         die $err;
5549     }
5550
5551     rmtree $tmpdir;
5552
5553     rename $tmpfn, $conffile ||
5554         die "unable to commit configuration file '$conffile'\n";
5555
5556     PVE::Cluster::cfs_update(); # make sure we read new file
5557
5558     eval { rescan($vmid, 1); };
5559     warn $@ if $@;
5560 };
5561
5562 sub foreach_writable_storage {
5563     my ($conf, $func) = @_;
5564
5565     my $sidhash = {};
5566
5567     foreach my $ds (keys %$conf) {
5568         next if !is_valid_drivename($ds);
5569
5570         my $drive = parse_drive($ds, $conf->{$ds});
5571         next if !$drive;
5572         next if drive_is_cdrom($drive);
5573
5574         my $volid = $drive->{file};
5575
5576         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5577         $sidhash->{$sid} = $sid if $sid;
5578     }
5579
5580     foreach my $sid (sort keys %$sidhash) {
5581         &$func($sid);
5582     }
5583 }
5584
5585 sub do_snapshots_with_qemu {
5586     my ($storecfg, $volid) = @_;
5587
5588     my $storage_name = PVE::Storage::parse_volume_id($volid);
5589
5590     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5591         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5592         return 1;
5593     }
5594
5595     if ($volid =~ m/\.(qcow2|qed)$/){
5596         return 1;
5597     }
5598
5599     return undef;
5600 }
5601
5602 sub qga_check_running {
5603     my ($vmid) = @_;
5604
5605     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5606     if ($@) {
5607         warn "Qemu Guest Agent are not running - $@";
5608         return 0;
5609     }
5610     return 1;
5611 }
5612
5613 sub template_create {
5614     my ($vmid, $conf, $disk) = @_;
5615
5616     my $storecfg = PVE::Storage::config();
5617
5618     foreach_drive($conf, sub {
5619         my ($ds, $drive) = @_;
5620
5621         return if drive_is_cdrom($drive);
5622         return if $disk && $ds ne $disk;
5623
5624         my $volid = $drive->{file};
5625         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5626
5627         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5628         $drive->{file} = $voliddst;
5629         $conf->{$ds} = print_drive($vmid, $drive);
5630         PVE::QemuConfig->write_config($vmid, $conf);
5631     });
5632 }
5633
5634 sub qemu_img_convert {
5635     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5636
5637     my $storecfg = PVE::Storage::config();
5638     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5639     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5640
5641     if ($src_storeid && $dst_storeid) {
5642
5643         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5644
5645         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5646         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5647
5648         my $src_format = qemu_img_format($src_scfg, $src_volname);
5649         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5650
5651         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5652         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5653
5654         my $cmd = [];
5655         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5656         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5657         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5658         if ($is_zero_initialized) {
5659             push @$cmd, "zeroinit:$dst_path";
5660         } else {
5661             push @$cmd, $dst_path;
5662         }
5663
5664         my $parser = sub {
5665             my $line = shift;
5666             if($line =~ m/\((\S+)\/100\%\)/){
5667                 my $percent = $1;
5668                 my $transferred = int($size * $percent / 100);
5669                 my $remaining = $size - $transferred;
5670
5671                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5672             }
5673
5674         };
5675
5676         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5677         my $err = $@;
5678         die "copy failed: $err" if $err;
5679     }
5680 }
5681
5682 sub qemu_img_format {
5683     my ($scfg, $volname) = @_;
5684
5685     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5686         return $1;
5687     } else {
5688         return "raw";
5689     }
5690 }
5691
5692 sub qemu_drive_mirror {
5693     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5694
5695     my $storecfg = PVE::Storage::config();
5696     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5697
5698     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5699
5700     my $format = qemu_img_format($dst_scfg, $dst_volname);
5701
5702     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5703
5704     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5705
5706     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5707     $opts->{format} = $format if $format;
5708
5709     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5710
5711     my $finish_job = sub {
5712         while (1) {
5713             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5714             my $stat = @$stats[0];
5715             last if !$stat;
5716             sleep 1;
5717         }
5718     };
5719
5720     eval {
5721     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5722         while (1) {
5723             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5724             my $stat = @$stats[0];
5725             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5726             die "error job is not mirroring" if $stat->{type} ne "mirror";
5727
5728             my $busy = $stat->{busy};
5729             my $ready = $stat->{ready};
5730
5731             if (my $total = $stat->{len}) {
5732                 my $transferred = $stat->{offset} || 0;
5733                 my $remaining = $total - $transferred;
5734                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5735
5736                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5737             }
5738
5739
5740             if ($stat->{ready} eq 'true') {
5741
5742                 last if $vmiddst != $vmid;
5743
5744                 # try to switch the disk if source and destination are on the same guest
5745                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5746                 if (!$@) {
5747                     &$finish_job();
5748                     last;
5749                 }
5750                 die $@ if $@ !~ m/cannot be completed/;
5751             }
5752             sleep 1;
5753         }
5754
5755
5756     };
5757     my $err = $@;
5758
5759     my $cancel_job = sub {
5760         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5761         &$finish_job();
5762     };
5763
5764     if ($err) {
5765         eval { &$cancel_job(); };
5766         die "mirroring error: $err";
5767     }
5768
5769     if ($vmiddst != $vmid) {
5770         # if we clone a disk for a new target vm, we don't switch the disk
5771         &$cancel_job(); # so we call block-job-cancel
5772     }
5773 }
5774
5775 sub clone_disk {
5776     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5777         $newvmid, $storage, $format, $full, $newvollist) = @_;
5778
5779     my $newvolid;
5780
5781     if (!$full) {
5782         print "create linked clone of drive $drivename ($drive->{file})\n";
5783         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5784         push @$newvollist, $newvolid;
5785     } else {
5786         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5787         $storeid = $storage if $storage;
5788
5789         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5790         if (!$format) {
5791             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5792             $format = qemu_img_format($scfg, $volname);
5793         }
5794
5795         # test if requested format is supported - else use default
5796         my $supported = grep { $_ eq $format } @$validFormats;
5797         $format = $defFormat if !$supported;
5798
5799         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5800
5801         print "create full clone of drive $drivename ($drive->{file})\n";
5802         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5803         push @$newvollist, $newvolid;
5804
5805         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
5806
5807         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5808         if (!$running || $snapname) {
5809             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5810         } else {
5811             #qemu 2.6
5812             die "drive-mirror is not working currently when iothread is enabled" if $drive->{iothread};
5813
5814             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5815         }
5816     }
5817
5818     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5819
5820     my $disk = $drive;
5821     $disk->{format} = undef;
5822     $disk->{file} = $newvolid;
5823     $disk->{size} = $size;
5824
5825     return $disk;
5826 }
5827
5828 # this only works if VM is running
5829 sub get_current_qemu_machine {
5830     my ($vmid) = @_;
5831
5832     my $cmd = { execute => 'query-machines', arguments => {} };
5833     my $res = vm_qmp_command($vmid, $cmd);
5834
5835     my ($current, $default);
5836     foreach my $e (@$res) {
5837         $default = $e->{name} if $e->{'is-default'};
5838         $current = $e->{name} if $e->{'is-current'};
5839     }
5840
5841     # fallback to the default machine if current is not supported by qemu
5842     return $current || $default || 'pc';
5843 }
5844
5845 sub qemu_machine_feature_enabled {
5846     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5847
5848     my $current_major;
5849     my $current_minor;
5850
5851     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5852
5853         $current_major = $3;
5854         $current_minor = $4;
5855
5856     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
5857
5858         $current_major = $1;
5859         $current_minor = $2;
5860     }
5861
5862     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
5863
5864
5865 }
5866
5867 sub qemu_machine_pxe {
5868     my ($vmid, $conf, $machine) = @_;
5869
5870     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
5871
5872     foreach my $opt (keys %$conf) {
5873         next if $opt !~ m/^net(\d+)$/;
5874         my $net = PVE::QemuServer::parse_net($conf->{$opt});
5875         next if !$net;
5876         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
5877         return $machine.".pxe" if $romfile =~ m/pxe/;
5878         last;
5879     }
5880
5881     return $machine;
5882 }
5883
5884 sub qemu_use_old_bios_files {
5885     my ($machine_type) = @_;
5886
5887     return if !$machine_type;
5888
5889     my $use_old_bios_files = undef;
5890
5891     if ($machine_type =~ m/^(\S+)\.pxe$/) {
5892         $machine_type = $1;
5893         $use_old_bios_files = 1;
5894     } else {
5895         my $kvmver = kvm_user_version();
5896         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
5897         # load new efi bios files on migration. So this hack is required to allow
5898         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
5899         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
5900         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
5901     }
5902
5903     return ($use_old_bios_files, $machine_type);
5904 }
5905
5906 sub lspci {
5907
5908     my $devices = {};
5909
5910     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
5911             my (undef, $id, $function) = @_;
5912             my $res = { id => $id, function => $function};
5913             push @{$devices->{$id}}, $res;
5914     });
5915
5916     return $devices;
5917 }
5918
5919 sub vm_iothreads_list {
5920     my ($vmid) = @_;
5921
5922     my $res = vm_mon_cmd($vmid, 'query-iothreads');
5923
5924     my $iothreads = {};
5925     foreach my $iothread (@$res) {
5926         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
5927     }
5928
5929     return $iothreads;
5930 }
5931
5932 sub scsihw_infos {
5933     my ($conf, $drive) = @_;
5934
5935     my $maxdev = 0;
5936
5937     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
5938         $maxdev = 7;
5939     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
5940         $maxdev = 1;
5941     } else {
5942         $maxdev = 256;
5943     }
5944
5945     my $controller = int($drive->{index} / $maxdev);
5946     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
5947
5948     return ($maxdev, $controller, $controller_prefix);
5949 }
5950
5951 # bash completion helper
5952
5953 sub complete_backup_archives {
5954     my ($cmdname, $pname, $cvalue) = @_;
5955
5956     my $cfg = PVE::Storage::config();
5957
5958     my $storeid;
5959
5960     if ($cvalue =~ m/^([^:]+):/) {
5961         $storeid = $1;
5962     }
5963
5964     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
5965
5966     my $res = [];
5967     foreach my $id (keys %$data) {
5968         foreach my $item (@{$data->{$id}}) {
5969             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
5970             push @$res, $item->{volid} if defined($item->{volid});
5971         }
5972     }
5973
5974     return $res;
5975 }
5976
5977 my $complete_vmid_full = sub {
5978     my ($running) = @_;
5979
5980     my $idlist = vmstatus();
5981
5982     my $res = [];
5983
5984     foreach my $id (keys %$idlist) {
5985         my $d = $idlist->{$id};
5986         if (defined($running)) {
5987             next if $d->{template};
5988             next if $running && $d->{status} ne 'running';
5989             next if !$running && $d->{status} eq 'running';
5990         }
5991         push @$res, $id;
5992
5993     }
5994     return $res;
5995 };
5996
5997 sub complete_vmid {
5998     return &$complete_vmid_full();
5999 }
6000
6001 sub complete_vmid_stopped {
6002     return &$complete_vmid_full(0);
6003 }
6004
6005 sub complete_vmid_running {
6006     return &$complete_vmid_full(1);
6007 }
6008
6009 sub complete_storage {
6010
6011     my $cfg = PVE::Storage::config();
6012     my $ids = $cfg->{ids};
6013
6014     my $res = [];
6015     foreach my $sid (keys %$ids) {
6016         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6017         next if !$ids->{$sid}->{content}->{images};
6018         push @$res, $sid;
6019     }
6020
6021     return $res;
6022 }
6023
6024 1;