PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::GuestHelpers;
  34 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  35 use PVE::QemuServer::Memory;
  36 use PVE::QemuServer::USB qw(parse_usb_device);
  37 use PVE::QemuServer::Cloudinit;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use Time::HiRes qw(gettimeofday);
  41 use File::Copy qw(copy);
  42 use URI::Escape;
  43
  44 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  45 my $OVMF = {
  46     x86_64 => [
  47         "$EDK2_FW_BASE/OVMF_CODE.fd",
  48         "$EDK2_FW_BASE/OVMF_VARS.fd"
  49     ],
  50     aarch64 => [
  51         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  52         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  53     ],
  54 };
  55
  56 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  57
  58 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  59
  60 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  61
  62 # Note about locking: we use flock on the config file protect
  63 # against concurent actions.
  64 # Aditionaly, we have a 'lock' setting in the config file. This
  65 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  66 # allowed when such lock is set. But you can ignore this kind of
  67 # lock with the --skiplock flag.
  68
  69 cfs_register_file('/qemu-server/',
  70                   \&parse_vm_config,
  71                   \&write_vm_config);
  72
  73 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  74     description => "Some command save/restore state from this location.",
  75     type => 'string',
  76     maxLength => 128,
  77     optional => 1,
  78 });
  79
  80 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  81     description => "The name of the snapshot.",
  82     type => 'string', format => 'pve-configid',
  83     maxLength => 40,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 mkdir "/etc/pve/nodes/$nodename";
 114 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 115 mkdir $confdir;
 116
 117 my $var_run_tmpdir = "/var/run/qemu-server";
 118 mkdir $var_run_tmpdir;
 119
 120 my $lock_dir = "/var/lock/qemu-server";
 121 mkdir $lock_dir;
 122
 123 my $cpu_vendor_list = {
 124     # Intel CPUs
 125     486 => 'GenuineIntel',
 126     pentium => 'GenuineIntel',
 127     pentium2  => 'GenuineIntel',
 128     pentium3  => 'GenuineIntel',
 129     coreduo => 'GenuineIntel',
 130     core2duo => 'GenuineIntel',
 131     Conroe  => 'GenuineIntel',
 132     Penryn  => 'GenuineIntel',
 133     Nehalem  => 'GenuineIntel',
 134     'Nehalem-IBRS'  => 'GenuineIntel',
 135     Westmere => 'GenuineIntel',
 136     'Westmere-IBRS' => 'GenuineIntel',
 137     SandyBridge => 'GenuineIntel',
 138     'SandyBridge-IBRS' => 'GenuineIntel',
 139     IvyBridge => 'GenuineIntel',
 140     'IvyBridge-IBRS' => 'GenuineIntel',
 141     Haswell => 'GenuineIntel',
 142     'Haswell-IBRS' => 'GenuineIntel',
 143     'Haswell-noTSX' => 'GenuineIntel',
 144     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 145     Broadwell => 'GenuineIntel',
 146     'Broadwell-IBRS' => 'GenuineIntel',
 147     'Broadwell-noTSX' => 'GenuineIntel',
 148     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 149     'Skylake-Client' => 'GenuineIntel',
 150     'Skylake-Client-IBRS' => 'GenuineIntel',
 151     'Skylake-Server' => 'GenuineIntel',
 152     'Skylake-Server-IBRS' => 'GenuineIntel',
 153
 154     # AMD CPUs
 155     athlon => 'AuthenticAMD',
 156     phenom  => 'AuthenticAMD',
 157     Opteron_G1  => 'AuthenticAMD',
 158     Opteron_G2  => 'AuthenticAMD',
 159     Opteron_G3  => 'AuthenticAMD',
 160     Opteron_G4  => 'AuthenticAMD',
 161     Opteron_G5  => 'AuthenticAMD',
 162     EPYC => 'AuthenticAMD',
 163     'EPYC-IBPB' => 'AuthenticAMD',
 164
 165     # generic types, use vendor from host node
 166     host => 'default',
 167     kvm32 => 'default',
 168     kvm64 => 'default',
 169     qemu32 => 'default',
 170     qemu64 => 'default',
 171     max => 'default',
 172 };
 173
 174 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 175
 176 my $cpu_fmt = {
 177     cputype => {
 178         description => "Emulated CPU type.",
 179         type => 'string',
 180         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 181         default => 'kvm64',
 182         default_key => 1,
 183     },
 184     hidden => {
 185         description => "Do not identify as a KVM virtual machine.",
 186         type => 'boolean',
 187         optional => 1,
 188         default => 0
 189     },
 190     'hv-vendor-id' => {
 191         type => 'string',
 192         pattern => qr/[a-zA-Z0-9]{1,12}/,
 193         format_description => 'vendor-id',
 194         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 195         optional => 1,
 196     },
 197     flags => {
 198         description => "List of additional CPU flags separated by ';'."
 199                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 200                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 201         format_description => '+FLAG[;-FLAG...]',
 202         type => 'string',
 203         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 204         optional => 1,
 205     },
 206 };
 207
 208 my $watchdog_fmt = {
 209     model => {
 210         default_key => 1,
 211         type => 'string',
 212         enum => [qw(i6300esb ib700)],
 213         description => "Watchdog type to emulate.",
 214         default => 'i6300esb',
 215         optional => 1,
 216     },
 217     action => {
 218         type => 'string',
 219         enum => [qw(reset shutdown poweroff pause debug none)],
 220         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 221         optional => 1,
 222     },
 223 };
 224 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 225
 226 my $agent_fmt = {
 227     enabled => {
 228         description => "Enable/disable Qemu GuestAgent.",
 229         type => 'boolean',
 230         default => 0,
 231         default_key => 1,
 232     },
 233     fstrim_cloned_disks => {
 234         description => "Run fstrim after cloning/moving a disk.",
 235         type => 'boolean',
 236         optional => 1,
 237         default => 0
 238     },
 239 };
 240
 241 my $vga_fmt = {
 242     type => {
 243         description => "Select the VGA type.",
 244         type => 'string',
 245         default => 'std',
 246         optional => 1,
 247         default_key => 1,
 248         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 249     },
 250     memory => {
 251         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 252         type => 'integer',
 253         optional => 1,
 254         minimum => 4,
 255         maximum => 512,
 256     },
 257 };
 258
 259 my $ivshmem_fmt = {
 260     size => {
 261         type => 'integer',
 262         minimum => 1,
 263         description => "The size of the file in MB.",
 264     },
 265     name => {
 266         type => 'string',
 267         pattern => '[a-zA-Z0-9\-]+',
 268         optional => 1,
 269         format_description => 'string',
 270         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6/3.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string',
 396         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 397         pattern => '[acdn]{1,4}',
 398         default => 'cdn',
 399     },
 400     bootdisk => {
 401         optional => 1,
 402         type => 'string', format => 'pve-qm-bootdisk',
 403         description => "Enable booting from specified disk.",
 404         pattern => '(ide|sata|scsi|virtio)\d+',
 405     },
 406     smp => {
 407         optional => 1,
 408         type => 'integer',
 409         description => "The number of CPUs. Please use option -sockets instead.",
 410         minimum => 1,
 411         default => 1,
 412     },
 413     sockets => {
 414         optional => 1,
 415         type => 'integer',
 416         description => "The number of CPU sockets.",
 417         minimum => 1,
 418         default => 1,
 419     },
 420     cores => {
 421         optional => 1,
 422         type => 'integer',
 423         description => "The number of cores per socket.",
 424         minimum => 1,
 425         default => 1,
 426     },
 427     numa => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Enable/disable NUMA.",
 431         default => 0,
 432     },
 433     hugepages => {
 434         optional => 1,
 435         type => 'string',
 436         description => "Enable/disable hugepages memory.",
 437         enum => [qw(any 2 1024)],
 438     },
 439     vcpus => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Number of hotplugged vcpus.",
 443         minimum => 1,
 444         default => 0,
 445     },
 446     acpi => {
 447         optional => 1,
 448         type => 'boolean',
 449         description => "Enable/disable ACPI.",
 450         default => 1,
 451     },
 452     agent => {
 453         optional => 1,
 454         description => "Enable/disable Qemu GuestAgent and its properties.",
 455         type => 'string',
 456         format => $agent_fmt,
 457     },
 458     kvm => {
 459         optional => 1,
 460         type => 'boolean',
 461         description => "Enable/disable KVM hardware virtualization.",
 462         default => 1,
 463     },
 464     tdf => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable time drift fix.",
 468         default => 0,
 469     },
 470     localtime => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 474     },
 475     freeze => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 479     },
 480     vga => {
 481         optional => 1,
 482         type => 'string', format => $vga_fmt,
 483         description => "Configure the VGA hardware.",
 484         verbose_description => "Configure the VGA Hardware. If you want to use ".
 485             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 486             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 487             "is 'std' for all OS types besides some Windows versions (XP and " .
 488             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 489             "display server. For win* OS you can select how many independent " .
 490             "displays you want, Linux guests can add displays them self.\n".
 491             "You can also run without any graphic card, using a serial device as terminal.",
 492     },
 493     watchdog => {
 494         optional => 1,
 495         type => 'string', format => 'pve-qm-watchdog',
 496         description => "Create a virtual hardware watchdog device.",
 497         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 498             " (by a guest action), the watchdog must be periodically polled " .
 499             "by an agent inside the guest or else the watchdog will reset " .
 500             "the guest (or execute the respective action specified)",
 501     },
 502     startdate => {
 503         optional => 1,
 504         type => 'string',
 505         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 506         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 507         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 508         default => 'now',
 509     },
 510     startup =>  get_standard_option('pve-startup-order'),
 511     template => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Enable/disable Template.",
 515         default => 0,
 516     },
 517     args => {
 518         optional => 1,
 519         type => 'string',
 520         description => "Arbitrary arguments passed to kvm.",
 521         verbose_description => <<EODESCR,
 522 Arbitrary arguments passed to kvm, for example:
 523
 524 args: -no-reboot -no-hpet
 525
 526 NOTE: this option is for experts only.
 527 EODESCR
 528     },
 529     tablet => {
 530         optional => 1,
 531         type => 'boolean',
 532         default => 1,
 533         description => "Enable/disable the USB tablet device.",
 534         verbose_description => "Enable/disable the USB tablet device. This device is " .
 535             "usually needed to allow absolute mouse positioning with VNC. " .
 536             "Else the mouse runs out of sync with normal VNC clients. " .
 537             "If you're running lots of console-only guests on one host, " .
 538             "you may consider disabling this to save some context switches. " .
 539             "This is turned off by default if you use spice (-vga=qxl).",
 540     },
 541     migrate_speed => {
 542         optional => 1,
 543         type => 'integer',
 544         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 545         minimum => 0,
 546         default => 0,
 547     },
 548     migrate_downtime => {
 549         optional => 1,
 550         type => 'number',
 551         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 552         minimum => 0,
 553         default => 0.1,
 554     },
 555     cdrom => {
 556         optional => 1,
 557         type => 'string', format => 'pve-qm-ide',
 558         typetext => '<volume>',
 559         description => "This is an alias for option -ide2",
 560     },
 561     cpu => {
 562         optional => 1,
 563         description => "Emulated CPU type.",
 564         type => 'string',
 565         format => $cpu_fmt,
 566     },
 567     parent => get_standard_option('pve-snapshot-name', {
 568         optional => 1,
 569         description => "Parent snapshot name. This is used internally, and should not be modified.",
 570     }),
 571     snaptime => {
 572         optional => 1,
 573         description => "Timestamp for snapshots.",
 574         type => 'integer',
 575         minimum => 0,
 576     },
 577     vmstate => {
 578         optional => 1,
 579         type => 'string', format => 'pve-volume-id',
 580         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 581     },
 582     vmstatestorage => get_standard_option('pve-storage-id', {
 583         description => "Default storage for VM state volumes/files.",
 584         optional => 1,
 585     }),
 586     runningmachine => get_standard_option('pve-qemu-machine', {
 587         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 588     }),
 589     machine => get_standard_option('pve-qemu-machine'),
 590     arch => {
 591         description => "Virtual processor architecture. Defaults to the host.",
 592         optional => 1,
 593         type => 'string',
 594         enum => [qw(x86_64 aarch64)],
 595     },
 596     smbios1 => {
 597         description => "Specify SMBIOS type 1 fields.",
 598         type => 'string', format => 'pve-qm-smbios1',
 599         maxLength => 256,
 600         optional => 1,
 601     },
 602     protection => {
 603         optional => 1,
 604         type => 'boolean',
 605         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 606         default => 0,
 607     },
 608     bios => {
 609         optional => 1,
 610         type => 'string',
 611         enum => [ qw(seabios ovmf) ],
 612         description => "Select BIOS implementation.",
 613         default => 'seabios',
 614     },
 615     vmgenid => {
 616         type => 'string',
 617         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 618         format_description => 'UUID',
 619         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 620         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 621             " 128-bit integer value identifier to the guest OS. This allows to".
 622             " notify the guest operating system when the virtual machine is".
 623             " executed with a different configuration (e.g. snapshot execution".
 624             " or creation from a template). The guest operating system notices".
 625             " the change, and is then able to react as appropriate by marking".
 626             " its copies of distributed databases as dirty, re-initializing its".
 627             " random number generator, etc.\n".
 628             "Note that auto-creation only works when done throug API/CLI create".
 629             " or update methods, but not when manually editing the config file.",
 630         default => "1 (autogenerated)",
 631         optional => 1,
 632     },
 633     hookscript => {
 634         type => 'string',
 635         format => 'pve-volume-id',
 636         optional => 1,
 637         description => "Script that will be executed during various steps in the vms lifetime.",
 638     },
 639     ivshmem => {
 640         type => 'string',
 641         format => $ivshmem_fmt,
 642         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 643         optional => 1,
 644     }
 645 };
 646
 647 my $cicustom_fmt = {
 648     meta => {
 649         type => 'string',
 650         optional => 1,
 651         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 652         format => 'pve-volume-id',
 653         format_description => 'volume',
 654     },
 655     network => {
 656         type => 'string',
 657         optional => 1,
 658         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 659         format => 'pve-volume-id',
 660         format_description => 'volume',
 661     },
 662     user => {
 663         type => 'string',
 664         optional => 1,
 665         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 666         format => 'pve-volume-id',
 667         format_description => 'volume',
 668     },
 669 };
 670 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 671
 672 my $confdesc_cloudinit = {
 673     citype => {
 674         optional => 1,
 675         type => 'string',
 676         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 677         enum => ['configdrive2', 'nocloud'],
 678     },
 679     ciuser => {
 680         optional => 1,
 681         type => 'string',
 682         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 683     },
 684     cipassword => {
 685         optional => 1,
 686         type => 'string',
 687         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 688     },
 689     cicustom => {
 690         optional => 1,
 691         type => 'string',
 692         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 693         format => 'pve-qm-cicustom',
 694     },
 695     searchdomain => {
 696         optional => 1,
 697         type => 'string',
 698         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 699     },
 700     nameserver => {
 701         optional => 1,
 702         type => 'string', format => 'address-list',
 703         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 704     },
 705     sshkeys => {
 706         optional => 1,
 707         type => 'string',
 708         format => 'urlencoded',
 709         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 710     },
 711 };
 712
 713 # what about other qemu settings ?
 714 #cpu => 'string',
 715 #machine => 'string',
 716 #fda => 'file',
 717 #fdb => 'file',
 718 #mtdblock => 'file',
 719 #sd => 'file',
 720 #pflash => 'file',
 721 #snapshot => 'bool',
 722 #bootp => 'file',
 723 ##tftp => 'dir',
 724 ##smb => 'dir',
 725 #kernel => 'file',
 726 #append => 'string',
 727 #initrd => 'file',
 728 ##soundhw => 'string',
 729
 730 while (my ($k, $v) = each %$confdesc) {
 731     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 732 }
 733
 734 my $MAX_IDE_DISKS = 4;
 735 my $MAX_SCSI_DISKS = 14;
 736 my $MAX_VIRTIO_DISKS = 16;
 737 my $MAX_SATA_DISKS = 6;
 738 my $MAX_USB_DEVICES = 5;
 739 my $MAX_NETS = 32;
 740 my $MAX_UNUSED_DISKS = 256;
 741 my $MAX_HOSTPCI_DEVICES = 4;
 742 my $MAX_SERIAL_PORTS = 4;
 743 my $MAX_PARALLEL_PORTS = 3;
 744 my $MAX_NUMA = 8;
 745
 746 my $numa_fmt = {
 747     cpus => {
 748         type => "string",
 749         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 750         description => "CPUs accessing this NUMA node.",
 751         format_description => "id[-id];...",
 752     },
 753     memory => {
 754         type => "number",
 755         description => "Amount of memory this NUMA node provides.",
 756         optional => 1,
 757     },
 758     hostnodes => {
 759         type => "string",
 760         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 761         description => "Host NUMA nodes to use.",
 762         format_description => "id[-id];...",
 763         optional => 1,
 764     },
 765     policy => {
 766         type => 'string',
 767         enum => [qw(preferred bind interleave)],
 768         description => "NUMA allocation policy.",
 769         optional => 1,
 770     },
 771 };
 772 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 773 my $numadesc = {
 774     optional => 1,
 775     type => 'string', format => $numa_fmt,
 776     description => "NUMA topology.",
 777 };
 778 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 779
 780 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 781     $confdesc->{"numa$i"} = $numadesc;
 782 }
 783
 784 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 785                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 786                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 787 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 788
 789 my $net_fmt_bridge_descr = <<__EOD__;
 790 Bridge to attach the network device to. The Proxmox VE standard bridge
 791 is called 'vmbr0'.
 792
 793 If you do not specify a bridge, we create a kvm user (NATed) network
 794 device, which provides DHCP and DNS services. The following addresses
 795 are used:
 796
 797  10.0.2.2   Gateway
 798  10.0.2.3   DNS Server
 799  10.0.2.4   SMB Server
 800
 801 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 802 __EOD__
 803
 804 my $net_fmt = {
 805     macaddr  => get_standard_option('mac-addr', {
 806         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 807     }),
 808     model => {
 809         type => 'string',
 810         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 811         enum => $nic_model_list,
 812         default_key => 1,
 813     },
 814     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 815     bridge => {
 816         type => 'string',
 817         description => $net_fmt_bridge_descr,
 818         format_description => 'bridge',
 819         optional => 1,
 820     },
 821     queues => {
 822         type => 'integer',
 823         minimum => 0, maximum => 16,
 824         description => 'Number of packet queues to be used on the device.',
 825         optional => 1,
 826     },
 827     rate => {
 828         type => 'number',
 829         minimum => 0,
 830         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 831         optional => 1,
 832     },
 833     tag => {
 834         type => 'integer',
 835         minimum => 1, maximum => 4094,
 836         description => 'VLAN tag to apply to packets on this interface.',
 837         optional => 1,
 838     },
 839     trunks => {
 840         type => 'string',
 841         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 842         description => 'VLAN trunks to pass through this interface.',
 843         format_description => 'vlanid[;vlanid...]',
 844         optional => 1,
 845     },
 846     firewall => {
 847         type => 'boolean',
 848         description => 'Whether this interface should be protected by the firewall.',
 849         optional => 1,
 850     },
 851     link_down => {
 852         type => 'boolean',
 853         description => 'Whether this interface should be disconnected (like pulling the plug).',
 854         optional => 1,
 855     },
 856 };
 857
 858 my $netdesc = {
 859     optional => 1,
 860     type => 'string', format => $net_fmt,
 861     description => "Specify network devices.",
 862 };
 863
 864 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 865
 866 my $ipconfig_fmt = {
 867     ip => {
 868         type => 'string',
 869         format => 'pve-ipv4-config',
 870         format_description => 'IPv4Format/CIDR',
 871         description => 'IPv4 address in CIDR format.',
 872         optional => 1,
 873         default => 'dhcp',
 874     },
 875     gw => {
 876         type => 'string',
 877         format => 'ipv4',
 878         format_description => 'GatewayIPv4',
 879         description => 'Default gateway for IPv4 traffic.',
 880         optional => 1,
 881         requires => 'ip',
 882     },
 883     ip6 => {
 884         type => 'string',
 885         format => 'pve-ipv6-config',
 886         format_description => 'IPv6Format/CIDR',
 887         description => 'IPv6 address in CIDR format.',
 888         optional => 1,
 889         default => 'dhcp',
 890     },
 891     gw6 => {
 892         type => 'string',
 893         format => 'ipv6',
 894         format_description => 'GatewayIPv6',
 895         description => 'Default gateway for IPv6 traffic.',
 896         optional => 1,
 897         requires => 'ip6',
 898     },
 899 };
 900 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 901 my $ipconfigdesc = {
 902     optional => 1,
 903     type => 'string', format => 'pve-qm-ipconfig',
 904     description => <<'EODESCR',
 905 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 906
 907 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 908
 909 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 910 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 911
 912 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 913 EODESCR
 914 };
 915 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 916
 917 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 918     $confdesc->{"net$i"} = $netdesc;
 919     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 920 }
 921
 922 foreach my $key (keys %$confdesc_cloudinit) {
 923     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 924 }
 925
 926 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 927 sub verify_volume_id_or_qm_path {
 928     my ($volid, $noerr) = @_;
 929
 930     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 931         return $volid;
 932     }
 933
 934     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 935     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 936     if ($@) {
 937         return undef if $noerr;
 938         die $@;
 939     }
 940     return $volid;
 941 }
 942
 943 my $drivename_hash;
 944
 945 my %drivedesc_base = (
 946     volume => { alias => 'file' },
 947     file => {
 948         type => 'string',
 949         format => 'pve-volume-id-or-qm-path',
 950         default_key => 1,
 951         format_description => 'volume',
 952         description => "The drive's backing volume.",
 953     },
 954     media => {
 955         type => 'string',
 956         enum => [qw(cdrom disk)],
 957         description => "The drive's media type.",
 958         default => 'disk',
 959         optional => 1
 960     },
 961     cyls => {
 962         type => 'integer',
 963         description => "Force the drive's physical geometry to have a specific cylinder count.",
 964         optional => 1
 965     },
 966     heads => {
 967         type => 'integer',
 968         description => "Force the drive's physical geometry to have a specific head count.",
 969         optional => 1
 970     },
 971     secs => {
 972         type => 'integer',
 973         description => "Force the drive's physical geometry to have a specific sector count.",
 974         optional => 1
 975     },
 976     trans => {
 977         type => 'string',
 978         enum => [qw(none lba auto)],
 979         description => "Force disk geometry bios translation mode.",
 980         optional => 1,
 981     },
 982     snapshot => {
 983         type => 'boolean',
 984         description => "Controls qemu's snapshot mode feature."
 985             . " If activated, changes made to the disk are temporary and will"
 986             . " be discarded when the VM is shutdown.",
 987         optional => 1,
 988     },
 989     cache => {
 990         type => 'string',
 991         enum => [qw(none writethrough writeback unsafe directsync)],
 992         description => "The drive's cache mode",
 993         optional => 1,
 994     },
 995     format => get_standard_option('pve-qm-image-format'),
 996     size => {
 997         type => 'string',
 998         format => 'disk-size',
 999         format_description => 'DiskSize',
1000         description => "Disk size. This is purely informational and has no effect.",
1001         optional => 1,
1002     },
1003     backup => {
1004         type => 'boolean',
1005         description => "Whether the drive should be included when making backups.",
1006         optional => 1,
1007     },
1008     replicate => {
1009         type => 'boolean',
1010         description => 'Whether the drive should considered for replication jobs.',
1011         optional => 1,
1012         default => 1,
1013     },
1014     rerror => {
1015         type => 'string',
1016         enum => [qw(ignore report stop)],
1017         description => 'Read error action.',
1018         optional => 1,
1019     },
1020     werror => {
1021         type => 'string',
1022         enum => [qw(enospc ignore report stop)],
1023         description => 'Write error action.',
1024         optional => 1,
1025     },
1026     aio => {
1027         type => 'string',
1028         enum => [qw(native threads)],
1029         description => 'AIO type to use.',
1030         optional => 1,
1031     },
1032     discard => {
1033         type => 'string',
1034         enum => [qw(ignore on)],
1035         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1036         optional => 1,
1037     },
1038     detect_zeroes => {
1039         type => 'boolean',
1040         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1041         optional => 1,
1042     },
1043     serial => {
1044         type => 'string',
1045         format => 'urlencoded',
1046         format_description => 'serial',
1047         maxLength => 20*3, # *3 since it's %xx url enoded
1048         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1049         optional => 1,
1050     },
1051     shared => {
1052         type => 'boolean',
1053         description => 'Mark this locally-managed volume as available on all nodes',
1054         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1055         optional => 1,
1056         default => 0,
1057     }
1058 );
1059
1060 my %iothread_fmt = ( iothread => {
1061         type => 'boolean',
1062         description => "Whether to use iothreads for this drive",
1063         optional => 1,
1064 });
1065
1066 my %model_fmt = (
1067     model => {
1068         type => 'string',
1069         format => 'urlencoded',
1070         format_description => 'model',
1071         maxLength => 40*3, # *3 since it's %xx url enoded
1072         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1073         optional => 1,
1074     },
1075 );
1076
1077 my %queues_fmt = (
1078     queues => {
1079         type => 'integer',
1080         description => "Number of queues.",
1081         minimum => 2,
1082         optional => 1
1083     }
1084 );
1085
1086 my %scsiblock_fmt = (
1087     scsiblock => {
1088         type => 'boolean',
1089         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1090         optional => 1,
1091         default => 0,
1092     },
1093 );
1094
1095 my %ssd_fmt = (
1096     ssd => {
1097         type => 'boolean',
1098         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1099         optional => 1,
1100     },
1101 );
1102
1103 my %wwn_fmt = (
1104     wwn => {
1105         type => 'string',
1106         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1107         format_description => 'wwn',
1108         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1109         optional => 1,
1110     },
1111 );
1112
1113 my $add_throttle_desc = sub {
1114     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1115     my $d = {
1116         type => $type,
1117         format_description => $unit,
1118         description => "Maximum $what in $longunit.",
1119         optional => 1,
1120     };
1121     $d->{minimum} = $minimum if defined($minimum);
1122     $drivedesc_base{$key} = $d;
1123 };
1124 # throughput: (leaky bucket)
1125 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1126 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1127 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1128 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1129 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1130 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1131 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1132 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1133 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1134
1135 # pools: (pool of IO before throttling starts taking effect)
1136 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1137 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1138 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1139 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1140 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1141 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1142
1143 # burst lengths
1144 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1145 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1146 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1147 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1148 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1149 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1150
1151 # legacy support
1152 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1153 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1154 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1155 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1156
1157 my $ide_fmt = {
1158     %drivedesc_base,
1159     %model_fmt,
1160     %ssd_fmt,
1161     %wwn_fmt,
1162 };
1163 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1164
1165 my $idedesc = {
1166     optional => 1,
1167     type => 'string', format => $ide_fmt,
1168     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1169 };
1170 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1171
1172 my $scsi_fmt = {
1173     %drivedesc_base,
1174     %iothread_fmt,
1175     %queues_fmt,
1176     %scsiblock_fmt,
1177     %ssd_fmt,
1178     %wwn_fmt,
1179 };
1180 my $scsidesc = {
1181     optional => 1,
1182     type => 'string', format => $scsi_fmt,
1183     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1184 };
1185 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1186
1187 my $sata_fmt = {
1188     %drivedesc_base,
1189     %ssd_fmt,
1190     %wwn_fmt,
1191 };
1192 my $satadesc = {
1193     optional => 1,
1194     type => 'string', format => $sata_fmt,
1195     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1196 };
1197 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1198
1199 my $virtio_fmt = {
1200     %drivedesc_base,
1201     %iothread_fmt,
1202 };
1203 my $virtiodesc = {
1204     optional => 1,
1205     type => 'string', format => $virtio_fmt,
1206     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1207 };
1208 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1209
1210 my $alldrive_fmt = {
1211     %drivedesc_base,
1212     %iothread_fmt,
1213     %model_fmt,
1214     %queues_fmt,
1215     %scsiblock_fmt,
1216     %ssd_fmt,
1217     %wwn_fmt,
1218 };
1219
1220 my $efidisk_fmt = {
1221     volume => { alias => 'file' },
1222     file => {
1223         type => 'string',
1224         format => 'pve-volume-id-or-qm-path',
1225         default_key => 1,
1226         format_description => 'volume',
1227         description => "The drive's backing volume.",
1228     },
1229     format => get_standard_option('pve-qm-image-format'),
1230     size => {
1231         type => 'string',
1232         format => 'disk-size',
1233         format_description => 'DiskSize',
1234         description => "Disk size. This is purely informational and has no effect.",
1235         optional => 1,
1236     },
1237 };
1238
1239 my $efidisk_desc = {
1240     optional => 1,
1241     type => 'string', format => $efidisk_fmt,
1242     description => "Configure a Disk for storing EFI vars",
1243 };
1244
1245 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1246
1247 my $usb_fmt = {
1248     host => {
1249         default_key => 1,
1250         type => 'string', format => 'pve-qm-usb-device',
1251         format_description => 'HOSTUSBDEVICE|spice',
1252         description => <<EODESCR,
1253 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1254
1255  'bus-port(.port)*' (decimal numbers) or
1256  'vendor_id:product_id' (hexadeciaml numbers) or
1257  'spice'
1258
1259 You can use the 'lsusb -t' command to list existing usb devices.
1260
1261 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1262
1263 The value 'spice' can be used to add a usb redirection devices for spice.
1264 EODESCR
1265     },
1266     usb3 => {
1267         optional => 1,
1268         type => 'boolean',
1269         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1270         default => 0,
1271     },
1272 };
1273
1274 my $usbdesc = {
1275     optional => 1,
1276     type => 'string', format => $usb_fmt,
1277     description => "Configure an USB device (n is 0 to 4).",
1278 };
1279 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1280
1281 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1282 my $hostpci_fmt = {
1283     host => {
1284         default_key => 1,
1285         type => 'string',
1286         pattern => qr/$PCIRE(;$PCIRE)*/,
1287         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1288         description => <<EODESCR,
1289 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1290 of PCI virtual functions of the host. HOSTPCIID syntax is:
1291
1292 'bus:dev.func' (hexadecimal numbers)
1293
1294 You can us the 'lspci' command to list existing PCI devices.
1295 EODESCR
1296     },
1297     rombar => {
1298         type => 'boolean',
1299         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1300         optional => 1,
1301         default => 1,
1302     },
1303     romfile => {
1304         type => 'string',
1305         pattern => '[^,;]+',
1306         format_description => 'string',
1307         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1308         optional => 1,
1309     },
1310     pcie => {
1311         type => 'boolean',
1312         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1313         optional => 1,
1314         default => 0,
1315     },
1316     'x-vga' => {
1317         type => 'boolean',
1318         description =>  "Enable vfio-vga device support.",
1319         optional => 1,
1320         default => 0,
1321     },
1322     'mdev' => {
1323         type => 'string',
1324         format_description => 'string',
1325         pattern => '[^/\.:]+',
1326         optional => 1,
1327         description => <<EODESCR
1328 The type of mediated device to use.
1329 An instance of this type will be created on startup of the VM and
1330 will be cleaned up when the VM stops.
1331 EODESCR
1332     }
1333 };
1334 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1335
1336 my $hostpcidesc = {
1337         optional => 1,
1338         type => 'string', format => 'pve-qm-hostpci',
1339         description => "Map host PCI devices into guest.",
1340         verbose_description =>  <<EODESCR,
1341 Map host PCI devices into guest.
1342
1343 NOTE: This option allows direct access to host hardware. So it is no longer
1344 possible to migrate such machines - use with special care.
1345
1346 CAUTION: Experimental! User reported problems with this option.
1347 EODESCR
1348 };
1349 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1350
1351 my $serialdesc = {
1352         optional => 1,
1353         type => 'string',
1354         pattern => '(/dev/.+|socket)',
1355         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1356         verbose_description =>  <<EODESCR,
1357 Create a serial device inside the VM (n is 0 to 3), and pass through a
1358 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1359 host side (use 'qm terminal' to open a terminal connection).
1360
1361 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1362
1363 CAUTION: Experimental! User reported problems with this option.
1364 EODESCR
1365 };
1366
1367 my $paralleldesc= {
1368         optional => 1,
1369         type => 'string',
1370         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1371         description =>  "Map host parallel devices (n is 0 to 2).",
1372         verbose_description =>  <<EODESCR,
1373 Map host parallel devices (n is 0 to 2).
1374
1375 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1376
1377 CAUTION: Experimental! User reported problems with this option.
1378 EODESCR
1379 };
1380
1381 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1382     $confdesc->{"parallel$i"} = $paralleldesc;
1383 }
1384
1385 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1386     $confdesc->{"serial$i"} = $serialdesc;
1387 }
1388
1389 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1390     $confdesc->{"hostpci$i"} = $hostpcidesc;
1391 }
1392
1393 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1394     $drivename_hash->{"ide$i"} = 1;
1395     $confdesc->{"ide$i"} = $idedesc;
1396 }
1397
1398 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1399     $drivename_hash->{"sata$i"} = 1;
1400     $confdesc->{"sata$i"} = $satadesc;
1401 }
1402
1403 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1404     $drivename_hash->{"scsi$i"} = 1;
1405     $confdesc->{"scsi$i"} = $scsidesc ;
1406 }
1407
1408 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1409     $drivename_hash->{"virtio$i"} = 1;
1410     $confdesc->{"virtio$i"} = $virtiodesc;
1411 }
1412
1413 $drivename_hash->{efidisk0} = 1;
1414 $confdesc->{efidisk0} = $efidisk_desc;
1415
1416 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1417     $confdesc->{"usb$i"} = $usbdesc;
1418 }
1419
1420 my $unuseddesc = {
1421     optional => 1,
1422     type => 'string', format => 'pve-volume-id',
1423     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1424 };
1425
1426 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1427     $confdesc->{"unused$i"} = $unuseddesc;
1428 }
1429
1430 my $kvm_api_version = 0;
1431
1432 sub kvm_version {
1433     return $kvm_api_version if $kvm_api_version;
1434
1435     open my $fh, '<', '/dev/kvm'
1436         or return undef;
1437
1438     # 0xae00 => KVM_GET_API_VERSION
1439     $kvm_api_version = ioctl($fh, 0xae00, 0);
1440
1441     return $kvm_api_version;
1442 }
1443
1444 my $kvm_user_version;
1445
1446 sub kvm_user_version {
1447
1448     return $kvm_user_version if $kvm_user_version;
1449
1450     $kvm_user_version = 'unknown';
1451
1452     my $code = sub {
1453         my $line = shift;
1454         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1455             $kvm_user_version = $2;
1456         }
1457     };
1458
1459     eval { run_command("kvm -version", outfunc => $code); };
1460     warn $@ if $@;
1461
1462     return $kvm_user_version;
1463
1464 }
1465
1466 sub kernel_has_vhost_net {
1467     return -c '/dev/vhost-net';
1468 }
1469
1470 sub valid_drive_names {
1471     # order is important - used to autoselect boot disk
1472     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1473             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1474             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1475             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1476             'efidisk0');
1477 }
1478
1479 sub is_valid_drivename {
1480     my $dev = shift;
1481
1482     return defined($drivename_hash->{$dev});
1483 }
1484
1485 sub option_exists {
1486     my $key = shift;
1487     return defined($confdesc->{$key});
1488 }
1489
1490 sub nic_models {
1491     return $nic_model_list;
1492 }
1493
1494 sub os_list_description {
1495
1496     return {
1497         other => 'Other',
1498         wxp => 'Windows XP',
1499         w2k => 'Windows 2000',
1500         w2k3 =>, 'Windows 2003',
1501         w2k8 => 'Windows 2008',
1502         wvista => 'Windows Vista',
1503         win7 => 'Windows 7',
1504         win8 => 'Windows 8/2012',
1505         win10 => 'Windows 10/2016',
1506         l24 => 'Linux 2.4',
1507         l26 => 'Linux 2.6',
1508     };
1509 }
1510
1511 my $cdrom_path;
1512
1513 sub get_cdrom_path {
1514
1515     return  $cdrom_path if $cdrom_path;
1516
1517     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1518     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1519     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1520 }
1521
1522 sub get_iso_path {
1523     my ($storecfg, $vmid, $cdrom) = @_;
1524
1525     if ($cdrom eq 'cdrom') {
1526         return get_cdrom_path();
1527     } elsif ($cdrom eq 'none') {
1528         return '';
1529     } elsif ($cdrom =~ m|^/|) {
1530         return $cdrom;
1531     } else {
1532         return PVE::Storage::path($storecfg, $cdrom);
1533     }
1534 }
1535
1536 # try to convert old style file names to volume IDs
1537 sub filename_to_volume_id {
1538     my ($vmid, $file, $media) = @_;
1539
1540      if (!($file eq 'none' || $file eq 'cdrom' ||
1541           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1542
1543         return undef if $file =~ m|/|;
1544
1545         if ($media && $media eq 'cdrom') {
1546             $file = "local:iso/$file";
1547         } else {
1548             $file = "local:$vmid/$file";
1549         }
1550     }
1551
1552     return $file;
1553 }
1554
1555 sub verify_media_type {
1556     my ($opt, $vtype, $media) = @_;
1557
1558     return if !$media;
1559
1560     my $etype;
1561     if ($media eq 'disk') {
1562         $etype = 'images';
1563     } elsif ($media eq 'cdrom') {
1564         $etype = 'iso';
1565     } else {
1566         die "internal error";
1567     }
1568
1569     return if ($vtype eq $etype);
1570
1571     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1572 }
1573
1574 sub cleanup_drive_path {
1575     my ($opt, $storecfg, $drive) = @_;
1576
1577     # try to convert filesystem paths to volume IDs
1578
1579     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1580         ($drive->{file} !~ m|^/dev/.+|) &&
1581         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1582         ($drive->{file} !~ m/^\d+$/)) {
1583         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1584         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1585         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1586         verify_media_type($opt, $vtype, $drive->{media});
1587         $drive->{file} = $volid;
1588     }
1589
1590     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1591 }
1592
1593 sub parse_hotplug_features {
1594     my ($data) = @_;
1595
1596     my $res = {};
1597
1598     return $res if $data eq '0';
1599
1600     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1601
1602     foreach my $feature (PVE::Tools::split_list($data)) {
1603         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1604             $res->{$1} = 1;
1605         } else {
1606             die "invalid hotplug feature '$feature'\n";
1607         }
1608     }
1609     return $res;
1610 }
1611
1612 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1613 sub pve_verify_hotplug_features {
1614     my ($value, $noerr) = @_;
1615
1616     return $value if parse_hotplug_features($value);
1617
1618     return undef if $noerr;
1619
1620     die "unable to parse hotplug option\n";
1621 }
1622
1623 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1624 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1625 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1626 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1627 #        [,iothread=on][,serial=serial][,model=model]
1628
1629 sub parse_drive {
1630     my ($key, $data) = @_;
1631
1632     my ($interface, $index);
1633
1634     if ($key =~ m/^([^\d]+)(\d+)$/) {
1635         $interface = $1;
1636         $index = $2;
1637     } else {
1638         return undef;
1639     }
1640
1641     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1642                                      : $confdesc->{$key}->{format};
1643     if (!$desc) {
1644         warn "invalid drive key: $key\n";
1645         return undef;
1646     }
1647     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1648     return undef if !$res;
1649     $res->{interface} = $interface;
1650     $res->{index} = $index;
1651
1652     my $error = 0;
1653     foreach my $opt (qw(bps bps_rd bps_wr)) {
1654         if (my $bps = defined(delete $res->{$opt})) {
1655             if (defined($res->{"m$opt"})) {
1656                 warn "both $opt and m$opt specified\n";
1657                 ++$error;
1658                 next;
1659             }
1660             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1661         }
1662     }
1663
1664     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1665     for my $requirement (
1666         [mbps_max => 'mbps'],
1667         [mbps_rd_max => 'mbps_rd'],
1668         [mbps_wr_max => 'mbps_wr'],
1669         [miops_max => 'miops'],
1670         [miops_rd_max => 'miops_rd'],
1671         [miops_wr_max => 'miops_wr'],
1672         [bps_max_length => 'mbps_max'],
1673         [bps_rd_max_length => 'mbps_rd_max'],
1674         [bps_wr_max_length => 'mbps_wr_max'],
1675         [iops_max_length => 'iops_max'],
1676         [iops_rd_max_length => 'iops_rd_max'],
1677         [iops_wr_max_length => 'iops_wr_max']) {
1678         my ($option, $requires) = @$requirement;
1679         if ($res->{$option} && !$res->{$requires}) {
1680             warn "$option requires $requires\n";
1681             ++$error;
1682         }
1683     }
1684
1685     return undef if $error;
1686
1687     return undef if $res->{mbps_rd} && $res->{mbps};
1688     return undef if $res->{mbps_wr} && $res->{mbps};
1689     return undef if $res->{iops_rd} && $res->{iops};
1690     return undef if $res->{iops_wr} && $res->{iops};
1691
1692     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1693         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1694         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1695         return undef if $res->{interface} eq 'virtio';
1696     }
1697
1698     if (my $size = $res->{size}) {
1699         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1700     }
1701
1702     return $res;
1703 }
1704
1705 sub print_drive {
1706     my ($vmid, $drive) = @_;
1707     my $data = { %$drive };
1708     delete $data->{$_} for qw(index interface);
1709     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1710 }
1711
1712 sub scsi_inquiry {
1713     my($fh, $noerr) = @_;
1714
1715     my $SG_IO = 0x2285;
1716     my $SG_GET_VERSION_NUM = 0x2282;
1717
1718     my $versionbuf = "\x00" x 8;
1719     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1720     if (!$ret) {
1721         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1722         return undef;
1723     }
1724     my $version = unpack("I", $versionbuf);
1725     if ($version < 30000) {
1726         die "scsi generic interface too old\n"  if !$noerr;
1727         return undef;
1728     }
1729
1730     my $buf = "\x00" x 36;
1731     my $sensebuf = "\x00" x 8;
1732     my $cmd = pack("C x3 C x1", 0x12, 36);
1733
1734     # see /usr/include/scsi/sg.h
1735     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1736
1737     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1738                       length($sensebuf), 0, length($buf), $buf,
1739                       $cmd, $sensebuf, 6000);
1740
1741     $ret = ioctl($fh, $SG_IO, $packet);
1742     if (!$ret) {
1743         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1744         return undef;
1745     }
1746
1747     my @res = unpack($sg_io_hdr_t, $packet);
1748     if ($res[17] || $res[18]) {
1749         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1750         return undef;
1751     }
1752
1753     my $res = {};
1754     (my $byte0, my $byte1, $res->{vendor},
1755      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1756
1757     $res->{removable} = $byte1 & 128 ? 1 : 0;
1758     $res->{type} = $byte0 & 31;
1759
1760     return $res;
1761 }
1762
1763 sub path_is_scsi {
1764     my ($path) = @_;
1765
1766     my $fh = IO::File->new("+<$path") || return undef;
1767     my $res = scsi_inquiry($fh, 1);
1768     close($fh);
1769
1770     return $res;
1771 }
1772
1773 sub machine_type_is_q35 {
1774     my ($conf) = @_;
1775
1776     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1777 }
1778
1779 sub print_tabletdevice_full {
1780     my ($conf, $arch) = @_;
1781
1782     my $q35 = machine_type_is_q35($conf);
1783
1784     # we use uhci for old VMs because tablet driver was buggy in older qemu
1785     my $usbbus;
1786     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1787         $usbbus = 'ehci';
1788     } else {
1789         $usbbus = 'uhci';
1790     }
1791
1792     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1793 }
1794
1795 sub print_keyboarddevice_full {
1796     my ($conf, $arch, $machine) = @_;
1797
1798     return undef if $arch ne 'aarch64';
1799
1800     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1801 }
1802
1803 sub print_drivedevice_full {
1804     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1805
1806     my $device = '';
1807     my $maxdev = 0;
1808
1809     if ($drive->{interface} eq 'virtio') {
1810         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1811         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1812         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1813     } elsif ($drive->{interface} eq 'scsi') {
1814
1815         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1816         my $unit = $drive->{index} % $maxdev;
1817         my $devicetype = 'hd';
1818         my $path = '';
1819         if (drive_is_cdrom($drive)) {
1820             $devicetype = 'cd';
1821         } else {
1822             if ($drive->{file} =~ m|^/|) {
1823                 $path = $drive->{file};
1824                 if (my $info = path_is_scsi($path)) {
1825                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1826                         $devicetype = 'block';
1827                     } elsif ($info->{type} == 1) { # tape
1828                         $devicetype = 'generic';
1829                     }
1830                 }
1831             } else {
1832                  $path = PVE::Storage::path($storecfg, $drive->{file});
1833             }
1834
1835             if($path =~ m/^iscsi\:\/\//){
1836                 $devicetype = 'generic';
1837             }
1838         }
1839
1840         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1841             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1842         } else {
1843             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1844         }
1845
1846         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1847             $device .= ",rotation_rate=1";
1848         }
1849         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1850
1851     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1852         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1853         my $controller = int($drive->{index} / $maxdev);
1854         my $unit = $drive->{index} % $maxdev;
1855         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1856
1857         $device = "ide-$devicetype";
1858         if ($drive->{interface} eq 'ide') {
1859             $device .= ",bus=ide.$controller,unit=$unit";
1860         } else {
1861             $device .= ",bus=ahci$controller.$unit";
1862         }
1863         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1864
1865         if ($devicetype eq 'hd') {
1866             if (my $model = $drive->{model}) {
1867                 $model = URI::Escape::uri_unescape($model);
1868                 $device .= ",model=$model";
1869             }
1870             if ($drive->{ssd}) {
1871                 $device .= ",rotation_rate=1";
1872             }
1873         }
1874         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1875     } elsif ($drive->{interface} eq 'usb') {
1876         die "implement me";
1877         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1878     } else {
1879         die "unsupported interface type";
1880     }
1881
1882     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1883
1884     if (my $serial = $drive->{serial}) {
1885         $serial = URI::Escape::uri_unescape($serial);
1886         $device .= ",serial=$serial";
1887     }
1888
1889
1890     return $device;
1891 }
1892
1893 sub get_initiator_name {
1894     my $initiator;
1895
1896     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1897     while (defined(my $line = <$fh>)) {
1898         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1899         $initiator = $1;
1900         last;
1901     }
1902     $fh->close();
1903
1904     return $initiator;
1905 }
1906
1907 sub print_drive_full {
1908     my ($storecfg, $vmid, $drive) = @_;
1909
1910     my $path;
1911     my $volid = $drive->{file};
1912     my $format;
1913
1914     if (drive_is_cdrom($drive)) {
1915         $path = get_iso_path($storecfg, $vmid, $volid);
1916     } else {
1917         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1918         if ($storeid) {
1919             $path = PVE::Storage::path($storecfg, $volid);
1920             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1921             $format = qemu_img_format($scfg, $volname);
1922         } else {
1923             $path = $volid;
1924             $format = "raw";
1925         }
1926    }
1927
1928     my $opts = '';
1929     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1930     foreach my $o (@qemu_drive_options) {
1931         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1932     }
1933
1934     # snapshot only accepts on|off
1935     if (defined($drive->{snapshot})) {
1936         my $v = $drive->{snapshot} ? 'on' : 'off';
1937         $opts .= ",snapshot=$v";
1938     }
1939
1940     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1941         my ($dir, $qmpname) = @$type;
1942         if (my $v = $drive->{"mbps$dir"}) {
1943             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1944         }
1945         if (my $v = $drive->{"mbps${dir}_max"}) {
1946             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1947         }
1948         if (my $v = $drive->{"bps${dir}_max_length"}) {
1949             $opts .= ",throttling.bps$qmpname-max-length=$v";
1950         }
1951         if (my $v = $drive->{"iops${dir}"}) {
1952             $opts .= ",throttling.iops$qmpname=$v";
1953         }
1954         if (my $v = $drive->{"iops${dir}_max"}) {
1955             $opts .= ",throttling.iops$qmpname-max=$v";
1956         }
1957         if (my $v = $drive->{"iops${dir}_max_length"}) {
1958             $opts .= ",throttling.iops$qmpname-max-length=$v";
1959         }
1960     }
1961
1962     $opts .= ",format=$format" if $format && !$drive->{format};
1963
1964     my $cache_direct = 0;
1965
1966     if (my $cache = $drive->{cache}) {
1967         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1968     } elsif (!drive_is_cdrom($drive)) {
1969         $opts .= ",cache=none";
1970         $cache_direct = 1;
1971     }
1972
1973     # aio native works only with O_DIRECT
1974     if (!$drive->{aio}) {
1975         if($cache_direct) {
1976             $opts .= ",aio=native";
1977         } else {
1978             $opts .= ",aio=threads";
1979         }
1980     }
1981
1982     if (!drive_is_cdrom($drive)) {
1983         my $detectzeroes;
1984         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1985             $detectzeroes = 'off';
1986         } elsif ($drive->{discard}) {
1987             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1988         } else {
1989             # This used to be our default with discard not being specified:
1990             $detectzeroes = 'on';
1991         }
1992         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1993     }
1994
1995     my $pathinfo = $path ? "file=$path," : '';
1996
1997     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1998 }
1999
2000 sub print_netdevice_full {
2001     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2002
2003     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2004
2005     my $device = $net->{model};
2006     if ($net->{model} eq 'virtio') {
2007          $device = 'virtio-net-pci';
2008      };
2009
2010     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2011     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2012     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2013         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2014         my $vectors = $net->{queues} * 2 + 2;
2015         $tmpstr .= ",vectors=$vectors,mq=on";
2016     }
2017     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2018
2019     if ($use_old_bios_files) {
2020         my $romfile;
2021         if ($device eq 'virtio-net-pci') {
2022             $romfile = 'pxe-virtio.rom';
2023         } elsif ($device eq 'e1000') {
2024             $romfile = 'pxe-e1000.rom';
2025         } elsif ($device eq 'ne2k') {
2026             $romfile = 'pxe-ne2k_pci.rom';
2027         } elsif ($device eq 'pcnet') {
2028             $romfile = 'pxe-pcnet.rom';
2029         } elsif ($device eq 'rtl8139') {
2030             $romfile = 'pxe-rtl8139.rom';
2031         }
2032         $tmpstr .= ",romfile=$romfile" if $romfile;
2033     }
2034
2035     return $tmpstr;
2036 }
2037
2038 sub print_netdev_full {
2039     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2040
2041     my $i = '';
2042     if ($netid =~ m/^net(\d+)$/) {
2043         $i = int($1);
2044     }
2045
2046     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2047
2048     my $ifname = "tap${vmid}i$i";
2049
2050     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2051     die "interface name '$ifname' is too long (max 15 character)\n"
2052         if length($ifname) >= 16;
2053
2054     my $vhostparam = '';
2055     if (is_native($arch)) {
2056         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2057     }
2058
2059     my $vmname = $conf->{name} || "vm$vmid";
2060
2061     my $netdev = "";
2062     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2063
2064     if ($net->{bridge}) {
2065         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2066     } else {
2067         $netdev = "type=user,id=$netid,hostname=$vmname";
2068     }
2069
2070     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2071
2072     return $netdev;
2073 }
2074
2075
2076 sub print_cpu_device {
2077     my ($conf, $id) = @_;
2078
2079     my $kvm = $conf->{kvm} // 1;
2080     my $cpu = $kvm ? "kvm64" : "qemu64";
2081     if (my $cputype = $conf->{cpu}) {
2082         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2083             or die "Cannot parse cpu description: $cputype\n";
2084         $cpu = $cpuconf->{cputype};
2085     }
2086
2087     my $cores = $conf->{cores} || 1;
2088
2089     my $current_core = ($id - 1) % $cores;
2090     my $current_socket = int(($id - 1 - $current_core)/$cores);
2091
2092     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2093 }
2094
2095 my $vga_map = {
2096     'cirrus' => 'cirrus-vga',
2097     'std' => 'VGA',
2098     'vmware' => 'vmware-svga',
2099     'virtio' => 'virtio-vga',
2100 };
2101
2102 sub print_vga_device {
2103     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2104
2105     my $type = $vga_map->{$vga->{type}};
2106     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2107         $type = 'virtio-gpu';
2108     }
2109     my $vgamem_mb = $vga->{memory};
2110     if ($qxlnum) {
2111         $type = $id ? 'qxl' : 'qxl-vga';
2112     }
2113     die "no devicetype for $vga->{type}\n" if !$type;
2114
2115     my $memory = "";
2116     if ($vgamem_mb) {
2117         if ($vga->{type} eq 'virtio') {
2118             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2119             $memory = ",max_hostmem=$bytes";
2120         } elsif ($qxlnum) {
2121             # from https://www.spice-space.org/multiple-monitors.html
2122             $memory = ",vgamem_mb=$vga->{memory}";
2123             my $ram = $vgamem_mb * 4;
2124             my $vram = $vgamem_mb * 2;
2125             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2126         } else {
2127             $memory = ",vgamem_mb=$vga->{memory}";
2128         }
2129     } elsif ($qxlnum && $id) {
2130         $memory = ",ram_size=67108864,vram_size=33554432";
2131     }
2132
2133     my $q35 = machine_type_is_q35($conf);
2134     my $vgaid = "vga" . ($id // '');
2135     my $pciaddr;
2136
2137     if ($q35 && $vgaid eq 'vga') {
2138         # the first display uses pcie.0 bus on q35 machines
2139         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2140     } else {
2141         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2142     }
2143
2144     return "$type,id=${vgaid}${memory}${pciaddr}";
2145 }
2146
2147 sub drive_is_cloudinit {
2148     my ($drive) = @_;
2149     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2150 }
2151
2152 sub drive_is_cdrom {
2153     my ($drive, $exclude_cloudinit) = @_;
2154
2155     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2156
2157     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2158
2159 }
2160
2161 sub parse_number_sets {
2162     my ($set) = @_;
2163     my $res = [];
2164     foreach my $part (split(/;/, $set)) {
2165         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2166             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2167             push @$res, [ $1, $2 ];
2168         } else {
2169             die "invalid range: $part\n";
2170         }
2171     }
2172     return $res;
2173 }
2174
2175 sub parse_numa {
2176     my ($data) = @_;
2177
2178     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2179     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2180     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2181     return $res;
2182 }
2183
2184 sub parse_hostpci {
2185     my ($value) = @_;
2186
2187     return undef if !$value;
2188
2189     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2190
2191     my @idlist = split(/;/, $res->{host});
2192     delete $res->{host};
2193     foreach my $id (@idlist) {
2194         if ($id =~ m/\./) { # full id 00:00.1
2195             push @{$res->{pciid}}, {
2196                 id => $id,
2197             };
2198         } else { # partial id 00:00
2199             $res->{pciid} = PVE::SysFSTools::lspci($id);
2200         }
2201     }
2202     return $res;
2203 }
2204
2205 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2206 sub parse_net {
2207     my ($data) = @_;
2208
2209     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2210     if ($@) {
2211         warn $@;
2212         return undef;
2213     }
2214     if (!defined($res->{macaddr})) {
2215         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2216         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2217     }
2218     return $res;
2219 }
2220
2221 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2222 sub parse_ipconfig {
2223     my ($data) = @_;
2224
2225     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2226     if ($@) {
2227         warn $@;
2228         return undef;
2229     }
2230
2231     if ($res->{gw} && !$res->{ip}) {
2232         warn 'gateway specified without specifying an IP address';
2233         return undef;
2234     }
2235     if ($res->{gw6} && !$res->{ip6}) {
2236         warn 'IPv6 gateway specified without specifying an IPv6 address';
2237         return undef;
2238     }
2239     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2240         warn 'gateway specified together with DHCP';
2241         return undef;
2242     }
2243     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2244         # gw6 + auto/dhcp
2245         warn "IPv6 gateway specified together with $res->{ip6} address";
2246         return undef;
2247     }
2248
2249     if (!$res->{ip} && !$res->{ip6}) {
2250         return { ip => 'dhcp', ip6 => 'dhcp' };
2251     }
2252
2253     return $res;
2254 }
2255
2256 sub print_net {
2257     my $net = shift;
2258
2259     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2260 }
2261
2262 sub add_random_macs {
2263     my ($settings) = @_;
2264
2265     foreach my $opt (keys %$settings) {
2266         next if $opt !~ m/^net(\d+)$/;
2267         my $net = parse_net($settings->{$opt});
2268         next if !$net;
2269         $settings->{$opt} = print_net($net);
2270     }
2271 }
2272
2273 sub vm_is_volid_owner {
2274     my ($storecfg, $vmid, $volid) = @_;
2275
2276     if ($volid !~  m|^/|) {
2277         my ($path, $owner);
2278         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2279         if ($owner && ($owner == $vmid)) {
2280             return 1;
2281         }
2282     }
2283
2284     return undef;
2285 }
2286
2287 sub split_flagged_list {
2288     my $text = shift || '';
2289     $text =~ s/[,;]/ /g;
2290     $text =~ s/^\s+//;
2291     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2292 }
2293
2294 sub join_flagged_list {
2295     my ($how, $lst) = @_;
2296     join $how, map { $lst->{$_} . $_ } keys %$lst;
2297 }
2298
2299 sub vmconfig_delete_pending_option {
2300     my ($conf, $key, $force) = @_;
2301
2302     delete $conf->{pending}->{$key};
2303     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2304     $pending_delete_hash->{$key} = $force ? '!' : '';
2305     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2306 }
2307
2308 sub vmconfig_undelete_pending_option {
2309     my ($conf, $key) = @_;
2310
2311     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2312     delete $pending_delete_hash->{$key};
2313
2314     if (%$pending_delete_hash) {
2315         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2316     } else {
2317         delete $conf->{pending}->{delete};
2318     }
2319 }
2320
2321 sub vmconfig_register_unused_drive {
2322     my ($storecfg, $vmid, $conf, $drive) = @_;
2323
2324     if (drive_is_cloudinit($drive)) {
2325         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2326         warn $@ if $@;
2327     } elsif (!drive_is_cdrom($drive)) {
2328         my $volid = $drive->{file};
2329         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2330             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2331         }
2332     }
2333 }
2334
2335 sub vmconfig_cleanup_pending {
2336     my ($conf) = @_;
2337
2338     # remove pending changes when nothing changed
2339     my $changes;
2340     foreach my $opt (keys %{$conf->{pending}}) {
2341         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2342             $changes = 1;
2343             delete $conf->{pending}->{$opt};
2344         }
2345     }
2346
2347     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2348     my $pending_delete_hash = {};
2349     while (my ($opt, $force) = each %$current_delete_hash) {
2350         if (defined($conf->{$opt})) {
2351             $pending_delete_hash->{$opt} = $force;
2352         } else {
2353             $changes = 1;
2354         }
2355     }
2356
2357     if (%$pending_delete_hash) {
2358         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2359     } else {
2360         delete $conf->{pending}->{delete};
2361     }
2362
2363     return $changes;
2364 }
2365
2366 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2367 my $smbios1_fmt = {
2368     uuid => {
2369         type => 'string',
2370         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2371         format_description => 'UUID',
2372         description => "Set SMBIOS1 UUID.",
2373         optional => 1,
2374     },
2375     version => {
2376         type => 'string',
2377         pattern => '\S+',
2378         format_description => 'string',
2379         description => "Set SMBIOS1 version.",
2380         optional => 1,
2381     },
2382     serial => {
2383         type => 'string',
2384         pattern => '\S+',
2385         format_description => 'string',
2386         description => "Set SMBIOS1 serial number.",
2387         optional => 1,
2388     },
2389     manufacturer => {
2390         type => 'string',
2391         pattern => '\S+',
2392         format_description => 'string',
2393         description => "Set SMBIOS1 manufacturer.",
2394         optional => 1,
2395     },
2396     product => {
2397         type => 'string',
2398         pattern => '\S+',
2399         format_description => 'string',
2400         description => "Set SMBIOS1 product ID.",
2401         optional => 1,
2402     },
2403     sku => {
2404         type => 'string',
2405         pattern => '\S+',
2406         format_description => 'string',
2407         description => "Set SMBIOS1 SKU string.",
2408         optional => 1,
2409     },
2410     family => {
2411         type => 'string',
2412         pattern => '\S+',
2413         format_description => 'string',
2414         description => "Set SMBIOS1 family string.",
2415         optional => 1,
2416     },
2417 };
2418
2419 sub parse_smbios1 {
2420     my ($data) = @_;
2421
2422     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2423     warn $@ if $@;
2424     return $res;
2425 }
2426
2427 sub print_smbios1 {
2428     my ($smbios1) = @_;
2429     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2430 }
2431
2432 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2433
2434 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2435 sub verify_bootdisk {
2436     my ($value, $noerr) = @_;
2437
2438     return $value if is_valid_drivename($value);
2439
2440     return undef if $noerr;
2441
2442     die "invalid boot disk '$value'\n";
2443 }
2444
2445 sub parse_watchdog {
2446     my ($value) = @_;
2447
2448     return undef if !$value;
2449
2450     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2451     warn $@ if $@;
2452     return $res;
2453 }
2454
2455 sub parse_guest_agent {
2456     my ($value) = @_;
2457
2458     return {} if !defined($value->{agent});
2459
2460     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2461     warn $@ if $@;
2462
2463     # if the agent is disabled ignore the other potentially set properties
2464     return {} if !$res->{enabled};
2465     return $res;
2466 }
2467
2468 sub parse_vga {
2469     my ($value) = @_;
2470
2471     return {} if !$value;
2472     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2473     warn $@ if $@;
2474     return $res;
2475 }
2476
2477 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2478 sub verify_usb_device {
2479     my ($value, $noerr) = @_;
2480
2481     return $value if parse_usb_device($value);
2482
2483     return undef if $noerr;
2484
2485     die "unable to parse usb device\n";
2486 }
2487
2488 # add JSON properties for create and set function
2489 sub json_config_properties {
2490     my $prop = shift;
2491
2492     foreach my $opt (keys %$confdesc) {
2493         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2494         $prop->{$opt} = $confdesc->{$opt};
2495     }
2496
2497     return $prop;
2498 }
2499
2500 # return copy of $confdesc_cloudinit to generate documentation
2501 sub cloudinit_config_properties {
2502
2503     return dclone($confdesc_cloudinit);
2504 }
2505
2506 sub check_type {
2507     my ($key, $value) = @_;
2508
2509     die "unknown setting '$key'\n" if !$confdesc->{$key};
2510
2511     my $type = $confdesc->{$key}->{type};
2512
2513     if (!defined($value)) {
2514         die "got undefined value\n";
2515     }
2516
2517     if ($value =~ m/[\n\r]/) {
2518         die "property contains a line feed\n";
2519     }
2520
2521     if ($type eq 'boolean') {
2522         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2523         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2524         die "type check ('boolean') failed - got '$value'\n";
2525     } elsif ($type eq 'integer') {
2526         return int($1) if $value =~ m/^(\d+)$/;
2527         die "type check ('integer') failed - got '$value'\n";
2528     } elsif ($type eq 'number') {
2529         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2530         die "type check ('number') failed - got '$value'\n";
2531     } elsif ($type eq 'string') {
2532         if (my $fmt = $confdesc->{$key}->{format}) {
2533             PVE::JSONSchema::check_format($fmt, $value);
2534             return $value;
2535         }
2536         $value =~ s/^\"(.*)\"$/$1/;
2537         return $value;
2538     } else {
2539         die "internal error"
2540     }
2541 }
2542
2543 sub touch_config {
2544     my ($vmid) = @_;
2545
2546     my $conf = PVE::QemuConfig->config_file($vmid);
2547     utime undef, undef, $conf;
2548 }
2549
2550 sub destroy_vm {
2551     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2552
2553     my $conffile = PVE::QemuConfig->config_file($vmid);
2554
2555     my $conf = PVE::QemuConfig->load_config($vmid);
2556
2557     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2558
2559     if ($conf->{template}) {
2560         # check if any base image is still used by a linked clone
2561         foreach_drive($conf, sub {
2562                 my ($ds, $drive) = @_;
2563
2564                 return if drive_is_cdrom($drive);
2565
2566                 my $volid = $drive->{file};
2567
2568                 return if !$volid || $volid =~ m|^/|;
2569
2570                 die "base volume '$volid' is still in use by linked cloned\n"
2571                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2572
2573         });
2574     }
2575
2576     # only remove disks owned by this VM
2577     foreach_drive($conf, sub {
2578         my ($ds, $drive) = @_;
2579
2580         return if drive_is_cdrom($drive, 1);
2581
2582         my $volid = $drive->{file};
2583
2584         return if !$volid || $volid =~ m|^/|;
2585
2586         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2587         return if !$path || !$owner || ($owner != $vmid);
2588
2589         eval {
2590             PVE::Storage::vdisk_free($storecfg, $volid);
2591         };
2592         warn "Could not remove disk '$volid', check manually: $@" if $@;
2593
2594     });
2595
2596     if ($keep_empty_config) {
2597         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2598     } else {
2599         unlink $conffile;
2600     }
2601
2602     # also remove unused disk
2603     eval {
2604         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2605
2606         eval {
2607             PVE::Storage::foreach_volid($dl, sub {
2608                 my ($volid, $sid, $volname, $d) = @_;
2609                 PVE::Storage::vdisk_free($storecfg, $volid);
2610             });
2611         };
2612         warn $@ if $@;
2613
2614     };
2615     warn $@ if $@;
2616 }
2617
2618 sub parse_vm_config {
2619     my ($filename, $raw) = @_;
2620
2621     return undef if !defined($raw);
2622
2623     my $res = {
2624         digest => Digest::SHA::sha1_hex($raw),
2625         snapshots => {},
2626         pending => {},
2627     };
2628
2629     $filename =~ m|/qemu-server/(\d+)\.conf$|
2630         || die "got strange filename '$filename'";
2631
2632     my $vmid = $1;
2633
2634     my $conf = $res;
2635     my $descr;
2636     my $section = '';
2637
2638     my @lines = split(/\n/, $raw);
2639     foreach my $line (@lines) {
2640         next if $line =~ m/^\s*$/;
2641
2642         if ($line =~ m/^\[PENDING\]\s*$/i) {
2643             $section = 'pending';
2644             if (defined($descr)) {
2645                 $descr =~ s/\s+$//;
2646                 $conf->{description} = $descr;
2647             }
2648             $descr = undef;
2649             $conf = $res->{$section} = {};
2650             next;
2651
2652         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2653             $section = $1;
2654             if (defined($descr)) {
2655                 $descr =~ s/\s+$//;
2656                 $conf->{description} = $descr;
2657             }
2658             $descr = undef;
2659             $conf = $res->{snapshots}->{$section} = {};
2660             next;
2661         }
2662
2663         if ($line =~ m/^\#(.*)\s*$/) {
2664             $descr = '' if !defined($descr);
2665             $descr .= PVE::Tools::decode_text($1) . "\n";
2666             next;
2667         }
2668
2669         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2670             $descr = '' if !defined($descr);
2671             $descr .= PVE::Tools::decode_text($2);
2672         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2673             $conf->{snapstate} = $1;
2674         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2675             my $key = $1;
2676             my $value = $2;
2677             $conf->{$key} = $value;
2678         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2679             my $value = $1;
2680             if ($section eq 'pending') {
2681                 $conf->{delete} = $value; # we parse this later
2682             } else {
2683                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2684             }
2685         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2686             my $key = $1;
2687             my $value = $2;
2688             eval { $value = check_type($key, $value); };
2689             if ($@) {
2690                 warn "vm $vmid - unable to parse value of '$key' - $@";
2691             } else {
2692                 $key = 'ide2' if $key eq 'cdrom';
2693                 my $fmt = $confdesc->{$key}->{format};
2694                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2695                     my $v = parse_drive($key, $value);
2696                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2697                         $v->{file} = $volid;
2698                         $value = print_drive($vmid, $v);
2699                     } else {
2700                         warn "vm $vmid - unable to parse value of '$key'\n";
2701                         next;
2702                     }
2703                 }
2704
2705                 $conf->{$key} = $value;
2706             }
2707         }
2708     }
2709
2710     if (defined($descr)) {
2711         $descr =~ s/\s+$//;
2712         $conf->{description} = $descr;
2713     }
2714     delete $res->{snapstate}; # just to be sure
2715
2716     return $res;
2717 }
2718
2719 sub write_vm_config {
2720     my ($filename, $conf) = @_;
2721
2722     delete $conf->{snapstate}; # just to be sure
2723
2724     if ($conf->{cdrom}) {
2725         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2726         $conf->{ide2} = $conf->{cdrom};
2727         delete $conf->{cdrom};
2728     }
2729
2730     # we do not use 'smp' any longer
2731     if ($conf->{sockets}) {
2732         delete $conf->{smp};
2733     } elsif ($conf->{smp}) {
2734         $conf->{sockets} = $conf->{smp};
2735         delete $conf->{cores};
2736         delete $conf->{smp};
2737     }
2738
2739     my $used_volids = {};
2740
2741     my $cleanup_config = sub {
2742         my ($cref, $pending, $snapname) = @_;
2743
2744         foreach my $key (keys %$cref) {
2745             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2746                 $key eq 'snapstate' || $key eq 'pending';
2747             my $value = $cref->{$key};
2748             if ($key eq 'delete') {
2749                 die "propertry 'delete' is only allowed in [PENDING]\n"
2750                     if !$pending;
2751                 # fixme: check syntax?
2752                 next;
2753             }
2754             eval { $value = check_type($key, $value); };
2755             die "unable to parse value of '$key' - $@" if $@;
2756
2757             $cref->{$key} = $value;
2758
2759             if (!$snapname && is_valid_drivename($key)) {
2760                 my $drive = parse_drive($key, $value);
2761                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2762             }
2763         }
2764     };
2765
2766     &$cleanup_config($conf);
2767
2768     &$cleanup_config($conf->{pending}, 1);
2769
2770     foreach my $snapname (keys %{$conf->{snapshots}}) {
2771         die "internal error" if $snapname eq 'pending';
2772         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2773     }
2774
2775     # remove 'unusedX' settings if we re-add a volume
2776     foreach my $key (keys %$conf) {
2777         my $value = $conf->{$key};
2778         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2779             delete $conf->{$key};
2780         }
2781     }
2782
2783     my $generate_raw_config = sub {
2784         my ($conf, $pending) = @_;
2785
2786         my $raw = '';
2787
2788         # add description as comment to top of file
2789         if (defined(my $descr = $conf->{description})) {
2790             if ($descr) {
2791                 foreach my $cl (split(/\n/, $descr)) {
2792                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2793                 }
2794             } else {
2795                 $raw .= "#\n" if $pending;
2796             }
2797         }
2798
2799         foreach my $key (sort keys %$conf) {
2800             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2801             $raw .= "$key: $conf->{$key}\n";
2802         }
2803         return $raw;
2804     };
2805
2806     my $raw = &$generate_raw_config($conf);
2807
2808     if (scalar(keys %{$conf->{pending}})){
2809         $raw .= "\n[PENDING]\n";
2810         $raw .= &$generate_raw_config($conf->{pending}, 1);
2811     }
2812
2813     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2814         $raw .= "\n[$snapname]\n";
2815         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2816     }
2817
2818     return $raw;
2819 }
2820
2821 sub load_defaults {
2822
2823     my $res = {};
2824
2825     # we use static defaults from our JSON schema configuration
2826     foreach my $key (keys %$confdesc) {
2827         if (defined(my $default = $confdesc->{$key}->{default})) {
2828             $res->{$key} = $default;
2829         }
2830     }
2831
2832     return $res;
2833 }
2834
2835 sub config_list {
2836     my $vmlist = PVE::Cluster::get_vmlist();
2837     my $res = {};
2838     return $res if !$vmlist || !$vmlist->{ids};
2839     my $ids = $vmlist->{ids};
2840
2841     foreach my $vmid (keys %$ids) {
2842         my $d = $ids->{$vmid};
2843         next if !$d->{node} || $d->{node} ne $nodename;
2844         next if !$d->{type} || $d->{type} ne 'qemu';
2845         $res->{$vmid}->{exists} = 1;
2846     }
2847     return $res;
2848 }
2849
2850 # test if VM uses local resources (to prevent migration)
2851 sub check_local_resources {
2852     my ($conf, $noerr) = @_;
2853
2854     my $loc_res = 0;
2855
2856     $loc_res = 1 if $conf->{hostusb}; # old syntax
2857     $loc_res = 1 if $conf->{hostpci}; # old syntax
2858
2859     $loc_res = 1 if $conf->{ivshmem};
2860
2861     foreach my $k (keys %$conf) {
2862         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2863         # sockets are safe: they will recreated be on the target side post-migrate
2864         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2865         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2866     }
2867
2868     die "VM uses local resources\n" if $loc_res && !$noerr;
2869
2870     return $loc_res;
2871 }
2872
2873 # check if used storages are available on all nodes (use by migrate)
2874 sub check_storage_availability {
2875     my ($storecfg, $conf, $node) = @_;
2876
2877     foreach_drive($conf, sub {
2878         my ($ds, $drive) = @_;
2879
2880         my $volid = $drive->{file};
2881         return if !$volid;
2882
2883         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2884         return if !$sid;
2885
2886         # check if storage is available on both nodes
2887         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2888         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2889    });
2890 }
2891
2892 # list nodes where all VM images are available (used by has_feature API)
2893 sub shared_nodes {
2894     my ($conf, $storecfg) = @_;
2895
2896     my $nodelist = PVE::Cluster::get_nodelist();
2897     my $nodehash = { map { $_ => 1 } @$nodelist };
2898     my $nodename = PVE::INotify::nodename();
2899
2900     foreach_drive($conf, sub {
2901         my ($ds, $drive) = @_;
2902
2903         my $volid = $drive->{file};
2904         return if !$volid;
2905
2906         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2907         if ($storeid) {
2908             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2909             if ($scfg->{disable}) {
2910                 $nodehash = {};
2911             } elsif (my $avail = $scfg->{nodes}) {
2912                 foreach my $node (keys %$nodehash) {
2913                     delete $nodehash->{$node} if !$avail->{$node};
2914                 }
2915             } elsif (!$scfg->{shared}) {
2916                 foreach my $node (keys %$nodehash) {
2917                     delete $nodehash->{$node} if $node ne $nodename
2918                 }
2919             }
2920         }
2921     });
2922
2923     return $nodehash
2924 }
2925
2926 sub check_cmdline {
2927     my ($pidfile, $pid) = @_;
2928
2929     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2930     if (defined($fh)) {
2931         my $line = <$fh>;
2932         $fh->close;
2933         return undef if !$line;
2934         my @param = split(/\0/, $line);
2935
2936         my $cmd = $param[0];
2937         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2938
2939         for (my $i = 0; $i < scalar (@param); $i++) {
2940             my $p = $param[$i];
2941             next if !$p;
2942             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2943                 my $p = $param[$i+1];
2944                 return 1 if $p && ($p eq $pidfile);
2945                 return undef;
2946             }
2947         }
2948     }
2949     return undef;
2950 }
2951
2952 sub check_running {
2953     my ($vmid, $nocheck, $node) = @_;
2954
2955     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2956
2957     die "unable to find configuration file for VM $vmid - no such machine\n"
2958         if !$nocheck && ! -f $filename;
2959
2960     my $pidfile = pidfile_name($vmid);
2961
2962     if (my $fd = IO::File->new("<$pidfile")) {
2963         my $st = stat($fd);
2964         my $line = <$fd>;
2965         close($fd);
2966
2967         my $mtime = $st->mtime;
2968         if ($mtime > time()) {
2969             warn "file '$filename' modified in future\n";
2970         }
2971
2972         if ($line =~ m/^(\d+)$/) {
2973             my $pid = $1;
2974             if (check_cmdline($pidfile, $pid)) {
2975                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2976                     return $pid;
2977                 }
2978             }
2979         }
2980     }
2981
2982     return undef;
2983 }
2984
2985 sub vzlist {
2986
2987     my $vzlist = config_list();
2988
2989     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2990
2991     while (defined(my $de = $fd->read)) {
2992         next if $de !~ m/^(\d+)\.pid$/;
2993         my $vmid = $1;
2994         next if !defined($vzlist->{$vmid});
2995         if (my $pid = check_running($vmid)) {
2996             $vzlist->{$vmid}->{pid} = $pid;
2997         }
2998     }
2999
3000     return $vzlist;
3001 }
3002
3003 sub disksize {
3004     my ($storecfg, $conf) = @_;
3005
3006     my $bootdisk = $conf->{bootdisk};
3007     return undef if !$bootdisk;
3008     return undef if !is_valid_drivename($bootdisk);
3009
3010     return undef if !$conf->{$bootdisk};
3011
3012     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3013     return undef if !defined($drive);
3014
3015     return undef if drive_is_cdrom($drive);
3016
3017     my $volid = $drive->{file};
3018     return undef if !$volid;
3019
3020     return $drive->{size};
3021 }
3022
3023 our $vmstatus_return_properties = {
3024     vmid => get_standard_option('pve-vmid'),
3025     status => {
3026         description => "Qemu process status.",
3027         type => 'string',
3028         enum => ['stopped', 'running'],
3029     },
3030     maxmem => {
3031         description => "Maximum memory in bytes.",
3032         type => 'integer',
3033         optional => 1,
3034         renderer => 'bytes',
3035     },
3036     maxdisk => {
3037         description => "Root disk size in bytes.",
3038         type => 'integer',
3039         optional => 1,
3040         renderer => 'bytes',
3041     },
3042     name => {
3043         description => "VM name.",
3044         type => 'string',
3045         optional => 1,
3046     },
3047     qmpstatus => {
3048         description => "Qemu QMP agent status.",
3049         type => 'string',
3050         optional => 1,
3051     },
3052     pid => {
3053         description => "PID of running qemu process.",
3054         type => 'integer',
3055         optional => 1,
3056     },
3057     uptime => {
3058         description => "Uptime.",
3059         type => 'integer',
3060         optional => 1,
3061         renderer => 'duration',
3062     },
3063     cpus => {
3064         description => "Maximum usable CPUs.",
3065         type => 'number',
3066         optional => 1,
3067     },
3068 };
3069
3070 my $last_proc_pid_stat;
3071
3072 # get VM status information
3073 # This must be fast and should not block ($full == false)
3074 # We only query KVM using QMP if $full == true (this can be slow)
3075 sub vmstatus {
3076     my ($opt_vmid, $full) = @_;
3077
3078     my $res = {};
3079
3080     my $storecfg = PVE::Storage::config();
3081
3082     my $list = vzlist();
3083     my $defaults = load_defaults();
3084
3085     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3086
3087     my $cpucount = $cpuinfo->{cpus} || 1;
3088
3089     foreach my $vmid (keys %$list) {
3090         next if $opt_vmid && ($vmid ne $opt_vmid);
3091
3092         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3093         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3094
3095         my $d = { vmid => $vmid };
3096         $d->{pid} = $list->{$vmid}->{pid};
3097
3098         # fixme: better status?
3099         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3100
3101         my $size = disksize($storecfg, $conf);
3102         if (defined($size)) {
3103             $d->{disk} = 0; # no info available
3104             $d->{maxdisk} = $size;
3105         } else {
3106             $d->{disk} = 0;
3107             $d->{maxdisk} = 0;
3108         }
3109
3110         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3111             * ($conf->{cores} || $defaults->{cores});
3112         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3113         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3114
3115         $d->{name} = $conf->{name} || "VM $vmid";
3116         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3117             : $defaults->{memory}*(1024*1024);
3118
3119         if ($conf->{balloon}) {
3120             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3121             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3122                 : $defaults->{shares};
3123         }
3124
3125         $d->{uptime} = 0;
3126         $d->{cpu} = 0;
3127         $d->{mem} = 0;
3128
3129         $d->{netout} = 0;
3130         $d->{netin} = 0;
3131
3132         $d->{diskread} = 0;
3133         $d->{diskwrite} = 0;
3134
3135         $d->{template} = PVE::QemuConfig->is_template($conf);
3136
3137         $d->{serial} = 1 if conf_has_serial($conf);
3138
3139         $res->{$vmid} = $d;
3140     }
3141
3142     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3143     foreach my $dev (keys %$netdev) {
3144         next if $dev !~ m/^tap([1-9]\d*)i/;
3145         my $vmid = $1;
3146         my $d = $res->{$vmid};
3147         next if !$d;
3148
3149         $d->{netout} += $netdev->{$dev}->{receive};
3150         $d->{netin} += $netdev->{$dev}->{transmit};
3151
3152         if ($full) {
3153             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3154             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3155         }
3156
3157     }
3158
3159     my $ctime = gettimeofday;
3160
3161     foreach my $vmid (keys %$list) {
3162
3163         my $d = $res->{$vmid};
3164         my $pid = $d->{pid};
3165         next if !$pid;
3166
3167         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3168         next if !$pstat; # not running
3169
3170         my $used = $pstat->{utime} + $pstat->{stime};
3171
3172         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3173
3174         if ($pstat->{vsize}) {
3175             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3176         }
3177
3178         my $old = $last_proc_pid_stat->{$pid};
3179         if (!$old) {
3180             $last_proc_pid_stat->{$pid} = {
3181                 time => $ctime,
3182                 used => $used,
3183                 cpu => 0,
3184             };
3185             next;
3186         }
3187
3188         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3189
3190         if ($dtime > 1000) {
3191             my $dutime = $used -  $old->{used};
3192
3193             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3194             $last_proc_pid_stat->{$pid} = {
3195                 time => $ctime,
3196                 used => $used,
3197                 cpu => $d->{cpu},
3198             };
3199         } else {
3200             $d->{cpu} = $old->{cpu};
3201         }
3202     }
3203
3204     return $res if !$full;
3205
3206     my $qmpclient = PVE::QMPClient->new();
3207
3208     my $ballooncb = sub {
3209         my ($vmid, $resp) = @_;
3210
3211         my $info = $resp->{'return'};
3212         return if !$info->{max_mem};
3213
3214         my $d = $res->{$vmid};
3215
3216         # use memory assigned to VM
3217         $d->{maxmem} = $info->{max_mem};
3218         $d->{balloon} = $info->{actual};
3219
3220         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3221             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3222             $d->{freemem} = $info->{free_mem};
3223         }
3224
3225         $d->{ballooninfo} = $info;
3226     };
3227
3228     my $blockstatscb = sub {
3229         my ($vmid, $resp) = @_;
3230         my $data = $resp->{'return'} || [];
3231         my $totalrdbytes = 0;
3232         my $totalwrbytes = 0;
3233
3234         for my $blockstat (@$data) {
3235             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3236             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3237
3238             $blockstat->{device} =~ s/drive-//;
3239             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3240         }
3241         $res->{$vmid}->{diskread} = $totalrdbytes;
3242         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3243     };
3244
3245     my $statuscb = sub {
3246         my ($vmid, $resp) = @_;
3247
3248         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3249         # this fails if ballon driver is not loaded, so this must be
3250         # the last commnand (following command are aborted if this fails).
3251         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3252
3253         my $status = 'unknown';
3254         if (!defined($status = $resp->{'return'}->{status})) {
3255             warn "unable to get VM status\n";
3256             return;
3257         }
3258
3259         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3260     };
3261
3262     foreach my $vmid (keys %$list) {
3263         next if $opt_vmid && ($vmid ne $opt_vmid);
3264         next if !$res->{$vmid}->{pid}; # not running
3265         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3266     }
3267
3268     $qmpclient->queue_execute(undef, 2);
3269
3270     foreach my $vmid (keys %$list) {
3271         next if $opt_vmid && ($vmid ne $opt_vmid);
3272         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3273     }
3274
3275     return $res;
3276 }
3277
3278 sub foreach_drive {
3279     my ($conf, $func, @param) = @_;
3280
3281     foreach my $ds (valid_drive_names()) {
3282         next if !defined($conf->{$ds});
3283
3284         my $drive = parse_drive($ds, $conf->{$ds});
3285         next if !$drive;
3286
3287         &$func($ds, $drive, @param);
3288     }
3289 }
3290
3291 sub foreach_volid {
3292     my ($conf, $func, @param) = @_;
3293
3294     my $volhash = {};
3295
3296     my $test_volid = sub {
3297         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3298
3299         return if !$volid;
3300
3301         $volhash->{$volid}->{cdrom} //= 1;
3302         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3303
3304         $volhash->{$volid}->{replicate} //= 0;
3305         $volhash->{$volid}->{replicate} = 1 if $replicate;
3306
3307         $volhash->{$volid}->{shared} //= 0;
3308         $volhash->{$volid}->{shared} = 1 if $shared;
3309
3310         $volhash->{$volid}->{referenced_in_config} //= 0;
3311         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3312
3313         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3314             if defined($snapname);
3315     };
3316
3317     foreach_drive($conf, sub {
3318         my ($ds, $drive) = @_;
3319         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3320     });
3321
3322     foreach my $snapname (keys %{$conf->{snapshots}}) {
3323         my $snap = $conf->{snapshots}->{$snapname};
3324         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3325         foreach_drive($snap, sub {
3326             my ($ds, $drive) = @_;
3327             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3328         });
3329     }
3330
3331     foreach my $volid (keys %$volhash) {
3332         &$func($volid, $volhash->{$volid}, @param);
3333     }
3334 }
3335
3336 sub conf_has_serial {
3337     my ($conf) = @_;
3338
3339     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3340         if ($conf->{"serial$i"}) {
3341             return 1;
3342         }
3343     }
3344
3345     return 0;
3346 }
3347
3348 sub vga_conf_has_spice {
3349     my ($vga) = @_;
3350
3351     my $vgaconf = parse_vga($vga);
3352     my $vgatype = $vgaconf->{type};
3353     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3354
3355     return $1 || 1;
3356 }
3357
3358 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3359 sub get_host_arch() {
3360     $host_arch = (POSIX::uname())[4] if !$host_arch;
3361     return $host_arch;
3362 }
3363
3364 sub is_native($) {
3365     my ($arch) = @_;
3366     return get_host_arch() eq $arch;
3367 }
3368
3369 my $default_machines = {
3370     x86_64 => 'pc',
3371     aarch64 => 'virt',
3372 };
3373
3374 sub get_basic_machine_info {
3375     my ($conf, $forcemachine) = @_;
3376
3377     my $arch = $conf->{arch} // get_host_arch();
3378     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3379     return ($arch, $machine);
3380 }
3381
3382 sub get_ovmf_files($) {
3383     my ($arch) = @_;
3384
3385     my $ovmf = $OVMF->{$arch}
3386         or die "no OVMF images known for architecture '$arch'\n";
3387
3388     return @$ovmf;
3389 }
3390
3391 my $Arch2Qemu = {
3392     aarch64 => '/usr/bin/qemu-system-aarch64',
3393     x86_64 => '/usr/bin/qemu-system-x86_64',
3394 };
3395 sub get_command_for_arch($) {
3396     my ($arch) = @_;
3397     return '/usr/bin/kvm' if is_native($arch);
3398
3399     my $cmd = $Arch2Qemu->{$arch}
3400         or die "don't know how to emulate architecture '$arch'\n";
3401     return $cmd;
3402 }
3403
3404 sub get_cpu_options {
3405     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3406
3407     my $cpuFlags = [];
3408     my $ostype = $conf->{ostype};
3409
3410     my $cpu = $kvm ? "kvm64" : "qemu64";
3411     if ($arch eq 'aarch64') {
3412         $cpu = 'cortex-a57';
3413     }
3414     my $hv_vendor_id;
3415     if (my $cputype = $conf->{cpu}) {
3416         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3417             or die "Cannot parse cpu description: $cputype\n";
3418         $cpu = $cpuconf->{cputype};
3419         $kvm_off = 1 if $cpuconf->{hidden};
3420         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3421
3422         if (defined(my $flags = $cpuconf->{flags})) {
3423             push @$cpuFlags, split(";", $flags);
3424         }
3425     }
3426
3427     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3428
3429     push @$cpuFlags , '-x2apic'
3430         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3431
3432     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3433
3434     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3435
3436     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3437
3438         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3439         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3440     }
3441
3442     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3443
3444     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3445
3446     push @$cpuFlags, 'kvm=off' if $kvm_off;
3447
3448     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3449         push @$cpuFlags, "vendor=${cpu_vendor}"
3450             if $cpu_vendor ne 'default';
3451     } elsif ($arch ne 'aarch64') {
3452         die "internal error"; # should not happen
3453     }
3454
3455     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3456
3457     return ('-cpu', $cpu);
3458 }
3459
3460 sub config_to_command {
3461     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3462
3463     my $cmd = [];
3464     my $globalFlags = [];
3465     my $machineFlags = [];
3466     my $rtcFlags = [];
3467     my $devices = [];
3468     my $pciaddr = '';
3469     my $bridges = {};
3470     my $kvmver = kvm_user_version();
3471     my $vernum = 0; # unknown
3472     my $ostype = $conf->{ostype};
3473     my $winversion = windows_version($ostype);
3474     my $kvm = $conf->{kvm};
3475
3476     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3477     $kvm //= 1 if is_native($arch);
3478
3479     if ($kvm) {
3480         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3481             if !defined kvm_version();
3482     }
3483
3484     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3485         $vernum = $1*1000000+$2*1000;
3486     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3487         $vernum = $1*1000000+$2*1000+$3;
3488     }
3489
3490     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3491
3492     my $have_ovz = -f '/proc/vz/vestat';
3493
3494     my $q35 = machine_type_is_q35($conf);
3495     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3496     my $use_old_bios_files = undef;
3497     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3498
3499     my $cpuunits = defined($conf->{cpuunits}) ?
3500             $conf->{cpuunits} : $defaults->{cpuunits};
3501
3502     push @$cmd, get_command_for_arch($arch);
3503
3504     push @$cmd, '-id', $vmid;
3505
3506     my $vmname = $conf->{name} || "vm$vmid";
3507
3508     push @$cmd, '-name', $vmname;
3509
3510     my $use_virtio = 0;
3511
3512     my $qmpsocket = qmp_socket($vmid);
3513     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3514     push @$cmd, '-mon', "chardev=qmp,mode=control";
3515
3516     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3517         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3518         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3519     }
3520
3521     push @$cmd, '-pidfile' , pidfile_name($vmid);
3522
3523     push @$cmd, '-daemonize';
3524
3525     if ($conf->{smbios1}) {
3526         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3527     }
3528
3529     if ($conf->{vmgenid}) {
3530         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3531     }
3532
3533     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3534     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3535         die "uefi base image not found\n" if ! -f $ovmf_code;
3536
3537         my $path;
3538         my $format;
3539         if (my $efidisk = $conf->{efidisk0}) {
3540             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3541             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3542             $format = $d->{format};
3543             if ($storeid) {
3544                 $path = PVE::Storage::path($storecfg, $d->{file});
3545                 if (!defined($format)) {
3546                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3547                     $format = qemu_img_format($scfg, $volname);
3548                 }
3549             } else {
3550                 $path = $d->{file};
3551                 die "efidisk format must be specified\n"
3552                     if !defined($format);
3553             }
3554         } else {
3555             warn "no efidisk configured! Using temporary efivars disk.\n";
3556             $path = "/tmp/$vmid-ovmf.fd";
3557             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3558             $format = 'raw';
3559         }
3560
3561         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3562         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3563     }
3564
3565
3566     # add usb controllers
3567     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3568     push @$devices, @usbcontrollers if @usbcontrollers;
3569     my $vga = parse_vga($conf->{vga});
3570
3571     my $qxlnum = vga_conf_has_spice($conf->{vga});
3572     $vga->{type} = 'qxl' if $qxlnum;
3573
3574     if (!$vga->{type}) {
3575         if ($arch eq 'aarch64') {
3576             $vga->{type} = 'virtio';
3577         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3578             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3579         } else {
3580             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3581         }
3582     }
3583
3584     # enable absolute mouse coordinates (needed by vnc)
3585     my $tablet;
3586     if (defined($conf->{tablet})) {
3587         $tablet = $conf->{tablet};
3588     } else {
3589         $tablet = $defaults->{tablet};
3590         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3591         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3592     }
3593
3594     if ($tablet) {
3595         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3596         my $kbd = print_keyboarddevice_full($conf, $arch);
3597         push @$devices, '-device', $kbd if defined($kbd);
3598     }
3599
3600     my $kvm_off = 0;
3601     my $gpu_passthrough;
3602
3603     # host pci devices
3604     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3605         my $d = parse_hostpci($conf->{"hostpci$i"});
3606         next if !$d;
3607
3608         my $pcie = $d->{pcie};
3609         if ($pcie) {
3610             die "q35 machine model is not enabled" if !$q35;
3611             # win7 wants to have the pcie devices directly on the pcie bus
3612             # instead of in the root port
3613             if ($winversion == 7) {
3614                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3615             } else {
3616                 $pciaddr = print_pcie_addr("hostpci$i");
3617             }
3618         } else {
3619             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3620         }
3621
3622         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3623         my $romfile = $d->{romfile};
3624
3625         my $xvga = '';
3626         if ($d->{'x-vga'}) {
3627             $xvga = ',x-vga=on';
3628             $kvm_off = 1;
3629             $vga->{type} = 'none' if !defined($conf->{vga});
3630             $gpu_passthrough = 1;
3631
3632             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3633                 $xvga = "";
3634             }
3635         }
3636         my $pcidevices = $d->{pciid};
3637         my $multifunction = 1 if @$pcidevices > 1;
3638         my $sysfspath;
3639         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3640             my $id = $pcidevices->[0]->{id};
3641             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3642             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3643         } elsif ($d->{mdev}) {
3644             warn "ignoring mediated device with multifunction device\n";
3645         }
3646
3647         my $j=0;
3648         foreach my $pcidevice (@$pcidevices) {
3649
3650             my $id = "hostpci$i";
3651             $id .= ".$j" if $multifunction;
3652             my $addr = $pciaddr;
3653             $addr .= ".$j" if $multifunction;
3654             my $devicestr = "vfio-pci";
3655             if ($sysfspath) {
3656                 $devicestr .= ",sysfsdev=$sysfspath";
3657             } else {
3658                 $devicestr .= ",host=$pcidevice->{id}";
3659             }
3660             $devicestr .= ",id=$id$addr";
3661
3662             if($j == 0){
3663                 $devicestr .= "$rombar$xvga";
3664                 $devicestr .= ",multifunction=on" if $multifunction;
3665                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3666             }
3667
3668             push @$devices, '-device', $devicestr;
3669             $j++;
3670         }
3671     }
3672
3673     # usb devices
3674     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3675     push @$devices, @usbdevices if @usbdevices;
3676     # serial devices
3677     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3678         if (my $path = $conf->{"serial$i"}) {
3679             if ($path eq 'socket') {
3680                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3681                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3682                 # On aarch64, serial0 is the UART device. Qemu only allows
3683                 # connecting UART devices via the '-serial' command line, as
3684                 # the device has a fixed slot on the hardware...
3685                 if ($arch eq 'aarch64' && $i == 0) {
3686                     push @$devices, '-serial', "chardev:serial$i";
3687                 } else {
3688                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3689                 }
3690             } else {
3691                 die "no such serial device\n" if ! -c $path;
3692                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3693                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3694             }
3695         }
3696     }
3697
3698     # parallel devices
3699     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3700         if (my $path = $conf->{"parallel$i"}) {
3701             die "no such parallel device\n" if ! -c $path;
3702             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3703             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3704             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3705         }
3706     }
3707
3708
3709     my $sockets = 1;
3710     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3711     $sockets = $conf->{sockets} if  $conf->{sockets};
3712
3713     my $cores = $conf->{cores} || 1;
3714
3715     my $maxcpus = $sockets * $cores;
3716
3717     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3718
3719     my $allowed_vcpus = $cpuinfo->{cpus};
3720
3721     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3722         if ($allowed_vcpus < $maxcpus);
3723
3724     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3725
3726         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3727         for (my $i = 2; $i <= $vcpus; $i++)  {
3728             my $cpustr = print_cpu_device($conf,$i);
3729             push @$cmd, '-device', $cpustr;
3730         }
3731
3732     } else {
3733
3734         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3735     }
3736     push @$cmd, '-nodefaults';
3737
3738     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3739
3740     my $bootindex_hash = {};
3741     my $i = 1;
3742     foreach my $o (split(//, $bootorder)) {
3743         $bootindex_hash->{$o} = $i*100;
3744         $i++;
3745     }
3746
3747     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3748
3749     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3750
3751     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3752
3753     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3754         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3755         my $socket = vnc_socket($vmid);
3756         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3757     } else {
3758         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3759         push @$cmd, '-nographic';
3760     }
3761
3762     # time drift fix
3763     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3764
3765     my $useLocaltime = $conf->{localtime};
3766
3767     if ($winversion >= 5) { # windows
3768         $useLocaltime = 1 if !defined($conf->{localtime});
3769
3770         # use time drift fix when acpi is enabled
3771         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3772             $tdf = 1 if !defined($conf->{tdf});
3773         }
3774     }
3775
3776     if ($winversion >= 6) {
3777         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3778         push @$cmd, '-no-hpet';
3779     }
3780
3781     push @$rtcFlags, 'driftfix=slew' if $tdf;
3782
3783     if (!$kvm) {
3784         push @$machineFlags, 'accel=tcg';
3785     }
3786
3787     if ($machine_type) {
3788         push @$machineFlags, "type=${machine_type}";
3789     }
3790
3791     if ($conf->{startdate}) {
3792         push @$rtcFlags, "base=$conf->{startdate}";
3793     } elsif ($useLocaltime) {
3794         push @$rtcFlags, 'base=localtime';
3795     }
3796
3797     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3798
3799     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3800
3801     push @$cmd, '-S' if $conf->{freeze};
3802
3803     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3804
3805     # enable sound
3806     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3807     #push @$cmd, '-soundhw', 'es1370';
3808     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3809
3810     if (parse_guest_agent($conf)->{enabled}) {
3811         my $qgasocket = qmp_socket($vmid, 1);
3812         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3813         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3814         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3815         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3816     }
3817
3818     my $spice_port;
3819
3820     if ($qxlnum) {
3821         if ($qxlnum > 1) {
3822             if ($winversion){
3823                 for(my $i = 1; $i < $qxlnum; $i++){
3824                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3825                 }
3826             } else {
3827                 # assume other OS works like Linux
3828                 my ($ram, $vram) = ("134217728", "67108864");
3829                 if ($vga->{memory}) {
3830                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3831                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3832                 }
3833                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3834                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3835             }
3836         }
3837
3838         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3839
3840         my $nodename = PVE::INotify::nodename();
3841         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3842         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3843         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3844         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3845         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3846
3847         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3848
3849         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3850         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3851         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3852     }
3853
3854     # enable balloon by default, unless explicitly disabled
3855     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3856         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3857         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3858     }
3859
3860     if ($conf->{watchdog}) {
3861         my $wdopts = parse_watchdog($conf->{watchdog});
3862         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3863         my $watchdog = $wdopts->{model} || 'i6300esb';
3864         push @$devices, '-device', "$watchdog$pciaddr";
3865         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3866     }
3867
3868     my $vollist = [];
3869     my $scsicontroller = {};
3870     my $ahcicontroller = {};
3871     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3872
3873     # Add iscsi initiator name if available
3874     if (my $initiator = get_initiator_name()) {
3875         push @$devices, '-iscsi', "initiator-name=$initiator";
3876     }
3877
3878     foreach_drive($conf, sub {
3879         my ($ds, $drive) = @_;
3880
3881         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3882             push @$vollist, $drive->{file};
3883         }
3884
3885         # ignore efidisk here, already added in bios/fw handling code above
3886         return if $drive->{interface} eq 'efidisk';
3887
3888         $use_virtio = 1 if $ds =~ m/^virtio/;
3889
3890         if (drive_is_cdrom ($drive)) {
3891             if ($bootindex_hash->{d}) {
3892                 $drive->{bootindex} = $bootindex_hash->{d};
3893                 $bootindex_hash->{d} += 1;
3894             }
3895         } else {
3896             if ($bootindex_hash->{c}) {
3897                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3898                 $bootindex_hash->{c} += 1;
3899             }
3900         }
3901
3902         if($drive->{interface} eq 'virtio'){
3903            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3904         }
3905
3906         if ($drive->{interface} eq 'scsi') {
3907
3908             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3909
3910             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3911             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3912
3913             my $iothread = '';
3914             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3915                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3916                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3917             } elsif ($drive->{iothread}) {
3918                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3919             }
3920
3921             my $queues = '';
3922             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3923                 $queues = ",num_queues=$drive->{queues}";
3924             }
3925
3926             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3927             $scsicontroller->{$controller}=1;
3928         }
3929
3930         if ($drive->{interface} eq 'sata') {
3931            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3932            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3933            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3934            $ahcicontroller->{$controller}=1;
3935         }
3936
3937         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3938         push @$devices, '-drive',$drive_cmd;
3939         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3940     });
3941
3942     for (my $i = 0; $i < $MAX_NETS; $i++) {
3943          next if !$conf->{"net$i"};
3944          my $d = parse_net($conf->{"net$i"});
3945          next if !$d;
3946
3947          $use_virtio = 1 if $d->{model} eq 'virtio';
3948
3949          if ($bootindex_hash->{n}) {
3950             $d->{bootindex} = $bootindex_hash->{n};
3951             $bootindex_hash->{n} += 1;
3952          }
3953
3954          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3955          push @$devices, '-netdev', $netdevfull;
3956
3957          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3958          push @$devices, '-device', $netdevicefull;
3959     }
3960
3961     if ($conf->{ivshmem}) {
3962         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3963
3964         my $bus;
3965         if ($q35) {
3966             $bus = print_pcie_addr("ivshmem");
3967         } else {
3968             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3969         }
3970
3971         my $ivshmem_name = $ivshmem->{name} // $vmid;
3972         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3973
3974         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3975         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3976     }
3977
3978     if (!$q35) {
3979         # add pci bridges
3980         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3981            $bridges->{1} = 1;
3982            $bridges->{2} = 1;
3983         }
3984
3985         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3986
3987         while (my ($k, $v) = each %$bridges) {
3988             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3989             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3990         }
3991     }
3992
3993     push @$cmd, @$devices;
3994     push @$cmd, '-rtc', join(',', @$rtcFlags)
3995         if scalar(@$rtcFlags);
3996     push @$cmd, '-machine', join(',', @$machineFlags)
3997         if scalar(@$machineFlags);
3998     push @$cmd, '-global', join(',', @$globalFlags)
3999         if scalar(@$globalFlags);
4000
4001     # add custom args
4002     if ($conf->{args}) {
4003         my $aa = PVE::Tools::split_args($conf->{args});
4004         push @$cmd, @$aa;
4005     }
4006
4007     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4008 }
4009
4010 sub vnc_socket {
4011     my ($vmid) = @_;
4012     return "${var_run_tmpdir}/$vmid.vnc";
4013 }
4014
4015 sub spice_port {
4016     my ($vmid) = @_;
4017
4018     my $res = vm_mon_cmd($vmid, 'query-spice');
4019
4020     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4021 }
4022
4023 sub qmp_socket {
4024     my ($vmid, $qga, $name) = @_;
4025     my $sockettype = $qga ? 'qga' : 'qmp';
4026     my $ext = $name ? '-'.$name : '';
4027     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4028 }
4029
4030 sub pidfile_name {
4031     my ($vmid) = @_;
4032     return "${var_run_tmpdir}/$vmid.pid";
4033 }
4034
4035 sub vm_devices_list {
4036     my ($vmid) = @_;
4037
4038     my $res = vm_mon_cmd($vmid, 'query-pci');
4039     my $devices_to_check = [];
4040     my $devices = {};
4041     foreach my $pcibus (@$res) {
4042         push @$devices_to_check, @{$pcibus->{devices}},
4043     }
4044
4045     while (@$devices_to_check) {
4046         my $to_check = [];
4047         for my $d (@$devices_to_check) {
4048             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4049             next if !$d->{'pci_bridge'};
4050
4051             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4052             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4053         }
4054         $devices_to_check = $to_check;
4055     }
4056
4057     my $resblock = vm_mon_cmd($vmid, 'query-block');
4058     foreach my $block (@$resblock) {
4059         if($block->{device} =~ m/^drive-(\S+)/){
4060                 $devices->{$1} = 1;
4061         }
4062     }
4063
4064     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4065     foreach my $mice (@$resmice) {
4066         if ($mice->{name} eq 'QEMU HID Tablet') {
4067             $devices->{tablet} = 1;
4068             last;
4069         }
4070     }
4071
4072     # for usb devices there is no query-usb
4073     # but we can iterate over the entries in
4074     # qom-list path=/machine/peripheral
4075     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4076     foreach my $per (@$resperipheral) {
4077         if ($per->{name} =~ m/^usb\d+$/) {
4078             $devices->{$per->{name}} = 1;
4079         }
4080     }
4081
4082     return $devices;
4083 }
4084
4085 sub vm_deviceplug {
4086     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4087
4088     my $q35 = machine_type_is_q35($conf);
4089
4090     my $devices_list = vm_devices_list($vmid);
4091     return 1 if defined($devices_list->{$deviceid});
4092
4093     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4094
4095     if ($deviceid eq 'tablet') {
4096
4097         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4098
4099     } elsif ($deviceid eq 'keyboard') {
4100
4101         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4102
4103     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4104
4105         die "usb hotplug currently not reliable\n";
4106         # since we can't reliably hot unplug all added usb devices
4107         # and usb passthrough disables live migration
4108         # we disable usb hotplugging for now
4109         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4110
4111     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4112
4113         qemu_iothread_add($vmid, $deviceid, $device);
4114
4115         qemu_driveadd($storecfg, $vmid, $device);
4116         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4117
4118         qemu_deviceadd($vmid, $devicefull);
4119         eval { qemu_deviceaddverify($vmid, $deviceid); };
4120         if (my $err = $@) {
4121             eval { qemu_drivedel($vmid, $deviceid); };
4122             warn $@ if $@;
4123             die $err;
4124         }
4125
4126     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4127
4128
4129         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4130         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4131         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4132
4133         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4134
4135         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4136             qemu_iothread_add($vmid, $deviceid, $device);
4137             $devicefull .= ",iothread=iothread-$deviceid";
4138         }
4139
4140         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4141             $devicefull .= ",num_queues=$device->{queues}";
4142         }
4143
4144         qemu_deviceadd($vmid, $devicefull);
4145         qemu_deviceaddverify($vmid, $deviceid);
4146
4147     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4148
4149         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4150         qemu_driveadd($storecfg, $vmid, $device);
4151
4152         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4153         eval { qemu_deviceadd($vmid, $devicefull); };
4154         if (my $err = $@) {
4155             eval { qemu_drivedel($vmid, $deviceid); };
4156             warn $@ if $@;
4157             die $err;
4158         }
4159
4160     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4161
4162         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4163
4164         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4165         my $use_old_bios_files = undef;
4166         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4167
4168         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4169         qemu_deviceadd($vmid, $netdevicefull);
4170         eval {
4171             qemu_deviceaddverify($vmid, $deviceid);
4172             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4173         };
4174         if (my $err = $@) {
4175             eval { qemu_netdevdel($vmid, $deviceid); };
4176             warn $@ if $@;
4177             die $err;
4178         }
4179
4180     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4181
4182         my $bridgeid = $2;
4183         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4184         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4185
4186         qemu_deviceadd($vmid, $devicefull);
4187         qemu_deviceaddverify($vmid, $deviceid);
4188
4189     } else {
4190         die "can't hotplug device '$deviceid'\n";
4191     }
4192
4193     return 1;
4194 }
4195
4196 # fixme: this should raise exceptions on error!
4197 sub vm_deviceunplug {
4198     my ($vmid, $conf, $deviceid) = @_;
4199
4200     my $devices_list = vm_devices_list($vmid);
4201     return 1 if !defined($devices_list->{$deviceid});
4202
4203     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4204
4205     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4206
4207         qemu_devicedel($vmid, $deviceid);
4208
4209     } elsif ($deviceid =~ m/^usb\d+$/) {
4210
4211         die "usb hotplug currently not reliable\n";
4212         # when unplugging usb devices this way,
4213         # there may be remaining usb controllers/hubs
4214         # so we disable it for now
4215         qemu_devicedel($vmid, $deviceid);
4216         qemu_devicedelverify($vmid, $deviceid);
4217
4218     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4219
4220         qemu_devicedel($vmid, $deviceid);
4221         qemu_devicedelverify($vmid, $deviceid);
4222         qemu_drivedel($vmid, $deviceid);
4223         qemu_iothread_del($conf, $vmid, $deviceid);
4224
4225     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4226
4227         qemu_devicedel($vmid, $deviceid);
4228         qemu_devicedelverify($vmid, $deviceid);
4229         qemu_iothread_del($conf, $vmid, $deviceid);
4230
4231     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4232
4233         qemu_devicedel($vmid, $deviceid);
4234         qemu_drivedel($vmid, $deviceid);
4235         qemu_deletescsihw($conf, $vmid, $deviceid);
4236
4237     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4238
4239         qemu_devicedel($vmid, $deviceid);
4240         qemu_devicedelverify($vmid, $deviceid);
4241         qemu_netdevdel($vmid, $deviceid);
4242
4243     } else {
4244         die "can't unplug device '$deviceid'\n";
4245     }
4246
4247     return 1;
4248 }
4249
4250 sub qemu_deviceadd {
4251     my ($vmid, $devicefull) = @_;
4252
4253     $devicefull = "driver=".$devicefull;
4254     my %options =  split(/[=,]/, $devicefull);
4255
4256     vm_mon_cmd($vmid, "device_add" , %options);
4257 }
4258
4259 sub qemu_devicedel {
4260     my ($vmid, $deviceid) = @_;
4261
4262     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4263 }
4264
4265 sub qemu_iothread_add {
4266     my($vmid, $deviceid, $device) = @_;
4267
4268     if ($device->{iothread}) {
4269         my $iothreads = vm_iothreads_list($vmid);
4270         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4271     }
4272 }
4273
4274 sub qemu_iothread_del {
4275     my($conf, $vmid, $deviceid) = @_;
4276
4277     my $confid = $deviceid;
4278     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4279         $confid = 'scsi' . $1;
4280     }
4281     my $device = parse_drive($confid, $conf->{$confid});
4282     if ($device->{iothread}) {
4283         my $iothreads = vm_iothreads_list($vmid);
4284         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4285     }
4286 }
4287
4288 sub qemu_objectadd {
4289     my($vmid, $objectid, $qomtype) = @_;
4290
4291     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4292
4293     return 1;
4294 }
4295
4296 sub qemu_objectdel {
4297     my($vmid, $objectid) = @_;
4298
4299     vm_mon_cmd($vmid, "object-del", id => $objectid);
4300
4301     return 1;
4302 }
4303
4304 sub qemu_driveadd {
4305     my ($storecfg, $vmid, $device) = @_;
4306
4307     my $drive = print_drive_full($storecfg, $vmid, $device);
4308     $drive =~ s/\\/\\\\/g;
4309     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4310
4311     # If the command succeeds qemu prints: "OK"
4312     return 1 if $ret =~ m/OK/s;
4313
4314     die "adding drive failed: $ret\n";
4315 }
4316
4317 sub qemu_drivedel {
4318     my($vmid, $deviceid) = @_;
4319
4320     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4321     $ret =~ s/^\s+//;
4322
4323     return 1 if $ret eq "";
4324
4325     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4326     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4327
4328     die "deleting drive $deviceid failed : $ret\n";
4329 }
4330
4331 sub qemu_deviceaddverify {
4332     my ($vmid, $deviceid) = @_;
4333
4334     for (my $i = 0; $i <= 5; $i++) {
4335          my $devices_list = vm_devices_list($vmid);
4336          return 1 if defined($devices_list->{$deviceid});
4337          sleep 1;
4338     }
4339
4340     die "error on hotplug device '$deviceid'\n";
4341 }
4342
4343
4344 sub qemu_devicedelverify {
4345     my ($vmid, $deviceid) = @_;
4346
4347     # need to verify that the device is correctly removed as device_del
4348     # is async and empty return is not reliable
4349
4350     for (my $i = 0; $i <= 5; $i++) {
4351          my $devices_list = vm_devices_list($vmid);
4352          return 1 if !defined($devices_list->{$deviceid});
4353          sleep 1;
4354     }
4355
4356     die "error on hot-unplugging device '$deviceid'\n";
4357 }
4358
4359 sub qemu_findorcreatescsihw {
4360     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4361
4362     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4363
4364     my $scsihwid="$controller_prefix$controller";
4365     my $devices_list = vm_devices_list($vmid);
4366
4367     if(!defined($devices_list->{$scsihwid})) {
4368         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4369     }
4370
4371     return 1;
4372 }
4373
4374 sub qemu_deletescsihw {
4375     my ($conf, $vmid, $opt) = @_;
4376
4377     my $device = parse_drive($opt, $conf->{$opt});
4378
4379     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4380         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4381         return 1;
4382     }
4383
4384     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4385
4386     my $devices_list = vm_devices_list($vmid);
4387     foreach my $opt (keys %{$devices_list}) {
4388         if (PVE::QemuServer::is_valid_drivename($opt)) {
4389             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4390             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4391                 return 1;
4392             }
4393         }
4394     }
4395
4396     my $scsihwid="scsihw$controller";
4397
4398     vm_deviceunplug($vmid, $conf, $scsihwid);
4399
4400     return 1;
4401 }
4402
4403 sub qemu_add_pci_bridge {
4404     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4405
4406     my $bridges = {};
4407
4408     my $bridgeid;
4409
4410     print_pci_addr($device, $bridges, $arch, $machine_type);
4411
4412     while (my ($k, $v) = each %$bridges) {
4413         $bridgeid = $k;
4414     }
4415     return 1 if !defined($bridgeid) || $bridgeid < 1;
4416
4417     my $bridge = "pci.$bridgeid";
4418     my $devices_list = vm_devices_list($vmid);
4419
4420     if (!defined($devices_list->{$bridge})) {
4421         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4422     }
4423
4424     return 1;
4425 }
4426
4427 sub qemu_set_link_status {
4428     my ($vmid, $device, $up) = @_;
4429
4430     vm_mon_cmd($vmid, "set_link", name => $device,
4431                up => $up ? JSON::true : JSON::false);
4432 }
4433
4434 sub qemu_netdevadd {
4435     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4436
4437     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4438     my %options =  split(/[=,]/, $netdev);
4439
4440     vm_mon_cmd($vmid, "netdev_add",  %options);
4441     return 1;
4442 }
4443
4444 sub qemu_netdevdel {
4445     my ($vmid, $deviceid) = @_;
4446
4447     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4448 }
4449
4450 sub qemu_usb_hotplug {
4451     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4452
4453     return if !$device;
4454
4455     # remove the old one first
4456     vm_deviceunplug($vmid, $conf, $deviceid);
4457
4458     # check if xhci controller is necessary and available
4459     if ($device->{usb3}) {
4460
4461         my $devicelist = vm_devices_list($vmid);
4462
4463         if (!$devicelist->{xhci}) {
4464             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4465             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4466         }
4467     }
4468     my $d = parse_usb_device($device->{host});
4469     $d->{usb3} = $device->{usb3};
4470
4471     # add the new one
4472     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4473 }
4474
4475 sub qemu_cpu_hotplug {
4476     my ($vmid, $conf, $vcpus) = @_;
4477
4478     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4479
4480     my $sockets = 1;
4481     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4482     $sockets = $conf->{sockets} if  $conf->{sockets};
4483     my $cores = $conf->{cores} || 1;
4484     my $maxcpus = $sockets * $cores;
4485
4486     $vcpus = $maxcpus if !$vcpus;
4487
4488     die "you can't add more vcpus than maxcpus\n"
4489         if $vcpus > $maxcpus;
4490
4491     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4492
4493     if ($vcpus < $currentvcpus) {
4494
4495         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4496
4497             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4498                 qemu_devicedel($vmid, "cpu$i");
4499                 my $retry = 0;
4500                 my $currentrunningvcpus = undef;
4501                 while (1) {
4502                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4503                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4504                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4505                     $retry++;
4506                     sleep 1;
4507                 }
4508                 #update conf after each succesfull cpu unplug
4509                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4510                 PVE::QemuConfig->write_config($vmid, $conf);
4511             }
4512         } else {
4513             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4514         }
4515
4516         return;
4517     }
4518
4519     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4520     die "vcpus in running vm does not match its configuration\n"
4521         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4522
4523     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4524
4525         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4526             my $cpustr = print_cpu_device($conf, $i);
4527             qemu_deviceadd($vmid, $cpustr);
4528
4529             my $retry = 0;
4530             my $currentrunningvcpus = undef;
4531             while (1) {
4532                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4533                 last if scalar(@{$currentrunningvcpus}) == $i;
4534                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4535                 sleep 1;
4536                 $retry++;
4537             }
4538             #update conf after each succesfull cpu hotplug
4539             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4540             PVE::QemuConfig->write_config($vmid, $conf);
4541         }
4542     } else {
4543
4544         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4545             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4546         }
4547     }
4548 }
4549
4550 sub qemu_block_set_io_throttle {
4551     my ($vmid, $deviceid,
4552         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4553         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4554         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4555         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4556
4557     return if !check_running($vmid) ;
4558
4559     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4560         bps => int($bps),
4561         bps_rd => int($bps_rd),
4562         bps_wr => int($bps_wr),
4563         iops => int($iops),
4564         iops_rd => int($iops_rd),
4565         iops_wr => int($iops_wr),
4566         bps_max => int($bps_max),
4567         bps_rd_max => int($bps_rd_max),
4568         bps_wr_max => int($bps_wr_max),
4569         iops_max => int($iops_max),
4570         iops_rd_max => int($iops_rd_max),
4571         iops_wr_max => int($iops_wr_max),
4572         bps_max_length => int($bps_max_length),
4573         bps_rd_max_length => int($bps_rd_max_length),
4574         bps_wr_max_length => int($bps_wr_max_length),
4575         iops_max_length => int($iops_max_length),
4576         iops_rd_max_length => int($iops_rd_max_length),
4577         iops_wr_max_length => int($iops_wr_max_length),
4578     );
4579
4580 }
4581
4582 # old code, only used to shutdown old VM after update
4583 sub __read_avail {
4584     my ($fh, $timeout) = @_;
4585
4586     my $sel = new IO::Select;
4587     $sel->add($fh);
4588
4589     my $res = '';
4590     my $buf;
4591
4592     my @ready;
4593     while (scalar (@ready = $sel->can_read($timeout))) {
4594         my $count;
4595         if ($count = $fh->sysread($buf, 8192)) {
4596             if ($buf =~ /^(.*)\(qemu\) $/s) {
4597                 $res .= $1;
4598                 last;
4599             } else {
4600                 $res .= $buf;
4601             }
4602         } else {
4603             if (!defined($count)) {
4604                 die "$!\n";
4605             }
4606             last;
4607         }
4608     }
4609
4610     die "monitor read timeout\n" if !scalar(@ready);
4611
4612     return $res;
4613 }
4614
4615 sub qemu_block_resize {
4616     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4617
4618     my $running = check_running($vmid);
4619
4620     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4621
4622     return if !$running;
4623
4624     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4625
4626 }
4627
4628 sub qemu_volume_snapshot {
4629     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4630
4631     my $running = check_running($vmid);
4632
4633     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4634         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4635     } else {
4636         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4637     }
4638 }
4639
4640 sub qemu_volume_snapshot_delete {
4641     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4642
4643     my $running = check_running($vmid);
4644
4645     if($running) {
4646
4647         $running = undef;
4648         my $conf = PVE::QemuConfig->load_config($vmid);
4649         foreach_drive($conf, sub {
4650             my ($ds, $drive) = @_;
4651             $running = 1 if $drive->{file} eq $volid;
4652         });
4653     }
4654
4655     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4656         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4657     } else {
4658         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4659     }
4660 }
4661
4662 sub set_migration_caps {
4663     my ($vmid) = @_;
4664
4665     my $cap_ref = [];
4666
4667     my $enabled_cap = {
4668         "auto-converge" => 1,
4669         "xbzrle" => 1,
4670         "x-rdma-pin-all" => 0,
4671         "zero-blocks" => 0,
4672         "compress" => 0
4673     };
4674
4675     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4676
4677     for my $supported_capability (@$supported_capabilities) {
4678         push @$cap_ref, {
4679             capability => $supported_capability->{capability},
4680             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4681         };
4682     }
4683
4684     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4685 }
4686
4687 my $fast_plug_option = {
4688     'lock' => 1,
4689     'name' => 1,
4690     'onboot' => 1,
4691     'shares' => 1,
4692     'startup' => 1,
4693     'description' => 1,
4694     'protection' => 1,
4695     'vmstatestorage' => 1,
4696     'hookscript' => 1,
4697 };
4698
4699 # hotplug changes in [PENDING]
4700 # $selection hash can be used to only apply specified options, for
4701 # example: { cores => 1 } (only apply changed 'cores')
4702 # $errors ref is used to return error messages
4703 sub vmconfig_hotplug_pending {
4704     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4705
4706     my $defaults = load_defaults();
4707     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4708
4709     # commit values which do not have any impact on running VM first
4710     # Note: those option cannot raise errors, we we do not care about
4711     # $selection and always apply them.
4712
4713     my $add_error = sub {
4714         my ($opt, $msg) = @_;
4715         $errors->{$opt} = "hotplug problem - $msg";
4716     };
4717
4718     my $changes = 0;
4719     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4720         if ($fast_plug_option->{$opt}) {
4721             $conf->{$opt} = $conf->{pending}->{$opt};
4722             delete $conf->{pending}->{$opt};
4723             $changes = 1;
4724         }
4725     }
4726
4727     if ($changes) {
4728         PVE::QemuConfig->write_config($vmid, $conf);
4729         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4730     }
4731
4732     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4733
4734     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4735     while (my ($opt, $force) = each %$pending_delete_hash) {
4736         next if $selection && !$selection->{$opt};
4737         eval {
4738             if ($opt eq 'hotplug') {
4739                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4740             } elsif ($opt eq 'tablet') {
4741                 die "skip\n" if !$hotplug_features->{usb};
4742                 if ($defaults->{tablet}) {
4743                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4744                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4745                         if $arch eq 'aarch64';
4746                 } else {
4747                     vm_deviceunplug($vmid, $conf, 'tablet');
4748                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4749                 }
4750             } elsif ($opt =~ m/^usb\d+/) {
4751                 die "skip\n";
4752                 # since we cannot reliably hot unplug usb devices
4753                 # we are disabling it
4754                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4755                 vm_deviceunplug($vmid, $conf, $opt);
4756             } elsif ($opt eq 'vcpus') {
4757                 die "skip\n" if !$hotplug_features->{cpu};
4758                 qemu_cpu_hotplug($vmid, $conf, undef);
4759             } elsif ($opt eq 'balloon') {
4760                 # enable balloon device is not hotpluggable
4761                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4762                 # here we reset the ballooning value to memory
4763                 my $balloon = $conf->{memory} || $defaults->{memory};
4764                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4765             } elsif ($fast_plug_option->{$opt}) {
4766                 # do nothing
4767             } elsif ($opt =~ m/^net(\d+)$/) {
4768                 die "skip\n" if !$hotplug_features->{network};
4769                 vm_deviceunplug($vmid, $conf, $opt);
4770             } elsif (is_valid_drivename($opt)) {
4771                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4772                 vm_deviceunplug($vmid, $conf, $opt);
4773                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4774             } elsif ($opt =~ m/^memory$/) {
4775                 die "skip\n" if !$hotplug_features->{memory};
4776                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4777             } elsif ($opt eq 'cpuunits') {
4778                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4779             } elsif ($opt eq 'cpulimit') {
4780                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4781             } else {
4782                 die "skip\n";
4783             }
4784         };
4785         if (my $err = $@) {
4786             &$add_error($opt, $err) if $err ne "skip\n";
4787         } else {
4788             # save new config if hotplug was successful
4789             delete $conf->{$opt};
4790             vmconfig_undelete_pending_option($conf, $opt);
4791             PVE::QemuConfig->write_config($vmid, $conf);
4792             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4793         }
4794     }
4795
4796     my $apply_pending_cloudinit;
4797     $apply_pending_cloudinit = sub {
4798         my ($key, $value) = @_;
4799         $apply_pending_cloudinit = sub {}; # once is enough
4800
4801         my @cloudinit_opts = keys %$confdesc_cloudinit;
4802         foreach my $opt (keys %{$conf->{pending}}) {
4803             next if !grep { $_ eq $opt } @cloudinit_opts;
4804             $conf->{$opt} = delete $conf->{pending}->{$opt};
4805         }
4806
4807         my $new_conf = { %$conf };
4808         $new_conf->{$key} = $value;
4809         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4810     };
4811
4812     foreach my $opt (keys %{$conf->{pending}}) {
4813         next if $selection && !$selection->{$opt};
4814         my $value = $conf->{pending}->{$opt};
4815         eval {
4816             if ($opt eq 'hotplug') {
4817                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4818             } elsif ($opt eq 'tablet') {
4819                 die "skip\n" if !$hotplug_features->{usb};
4820                 if ($value == 1) {
4821                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4822                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4823                         if $arch eq 'aarch64';
4824                 } elsif ($value == 0) {
4825                     vm_deviceunplug($vmid, $conf, 'tablet');
4826                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4827                 }
4828             } elsif ($opt =~ m/^usb\d+$/) {
4829                 die "skip\n";
4830                 # since we cannot reliably hot unplug usb devices
4831                 # we are disabling it
4832                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4833                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4834                 die "skip\n" if !$d;
4835                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4836             } elsif ($opt eq 'vcpus') {
4837                 die "skip\n" if !$hotplug_features->{cpu};
4838                 qemu_cpu_hotplug($vmid, $conf, $value);
4839             } elsif ($opt eq 'balloon') {
4840                 # enable/disable balloning device is not hotpluggable
4841                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4842                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4843                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4844
4845                 # allow manual ballooning if shares is set to zero
4846                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4847                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4848                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4849                 }
4850             } elsif ($opt =~ m/^net(\d+)$/) {
4851                 # some changes can be done without hotplug
4852                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4853                                     $vmid, $opt, $value, $arch, $machine_type);
4854             } elsif (is_valid_drivename($opt)) {
4855                 # some changes can be done without hotplug
4856                 my $drive = parse_drive($opt, $value);
4857                 if (drive_is_cloudinit($drive)) {
4858                     &$apply_pending_cloudinit($opt, $value);
4859                 }
4860                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4861                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4862             } elsif ($opt =~ m/^memory$/) { #dimms
4863                 die "skip\n" if !$hotplug_features->{memory};
4864                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4865             } elsif ($opt eq 'cpuunits') {
4866                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4867             } elsif ($opt eq 'cpulimit') {
4868                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4869                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4870             } else {
4871                 die "skip\n";  # skip non-hot-pluggable options
4872             }
4873         };
4874         if (my $err = $@) {
4875             &$add_error($opt, $err) if $err ne "skip\n";
4876         } else {
4877             # save new config if hotplug was successful
4878             $conf->{$opt} = $value;
4879             delete $conf->{pending}->{$opt};
4880             PVE::QemuConfig->write_config($vmid, $conf);
4881             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4882         }
4883     }
4884 }
4885
4886 sub try_deallocate_drive {
4887     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4888
4889     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4890         my $volid = $drive->{file};
4891         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4892             my $sid = PVE::Storage::parse_volume_id($volid);
4893             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4894
4895             # check if the disk is really unused
4896             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4897                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4898             PVE::Storage::vdisk_free($storecfg, $volid);
4899             return 1;
4900         } else {
4901             # If vm is not owner of this disk remove from config
4902             return 1;
4903         }
4904     }
4905
4906     return undef;
4907 }
4908
4909 sub vmconfig_delete_or_detach_drive {
4910     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4911
4912     my $drive = parse_drive($opt, $conf->{$opt});
4913
4914     my $rpcenv = PVE::RPCEnvironment::get();
4915     my $authuser = $rpcenv->get_user();
4916
4917     if ($force) {
4918         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4919         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4920     } else {
4921         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4922     }
4923 }
4924
4925 sub vmconfig_apply_pending {
4926     my ($vmid, $conf, $storecfg) = @_;
4927
4928     # cold plug
4929
4930     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4931     while (my ($opt, $force) = each %$pending_delete_hash) {
4932         die "internal error" if $opt =~ m/^unused/;
4933         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4934         if (!defined($conf->{$opt})) {
4935             vmconfig_undelete_pending_option($conf, $opt);
4936             PVE::QemuConfig->write_config($vmid, $conf);
4937         } elsif (is_valid_drivename($opt)) {
4938             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4939             vmconfig_undelete_pending_option($conf, $opt);
4940             delete $conf->{$opt};
4941             PVE::QemuConfig->write_config($vmid, $conf);
4942         } else {
4943             vmconfig_undelete_pending_option($conf, $opt);
4944             delete $conf->{$opt};
4945             PVE::QemuConfig->write_config($vmid, $conf);
4946         }
4947     }
4948
4949     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4950
4951     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4952         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4953
4954         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4955             # skip if nothing changed
4956         } elsif (is_valid_drivename($opt)) {
4957             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4958                 if defined($conf->{$opt});
4959             $conf->{$opt} = $conf->{pending}->{$opt};
4960         } else {
4961             $conf->{$opt} = $conf->{pending}->{$opt};
4962         }
4963
4964         delete $conf->{pending}->{$opt};
4965         PVE::QemuConfig->write_config($vmid, $conf);
4966     }
4967 }
4968
4969 my $safe_num_ne = sub {
4970     my ($a, $b) = @_;
4971
4972     return 0 if !defined($a) && !defined($b);
4973     return 1 if !defined($a);
4974     return 1 if !defined($b);
4975
4976     return $a != $b;
4977 };
4978
4979 my $safe_string_ne = sub {
4980     my ($a, $b) = @_;
4981
4982     return 0 if !defined($a) && !defined($b);
4983     return 1 if !defined($a);
4984     return 1 if !defined($b);
4985
4986     return $a ne $b;
4987 };
4988
4989 sub vmconfig_update_net {
4990     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4991
4992     my $newnet = parse_net($value);
4993
4994     if ($conf->{$opt}) {
4995         my $oldnet = parse_net($conf->{$opt});
4996
4997         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4998             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4999             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5000             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5001
5002             # for non online change, we try to hot-unplug
5003             die "skip\n" if !$hotplug;
5004             vm_deviceunplug($vmid, $conf, $opt);
5005         } else {
5006
5007             die "internal error" if $opt !~ m/net(\d+)/;
5008             my $iface = "tap${vmid}i$1";
5009
5010             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5011                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5012                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5013                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5014                 PVE::Network::tap_unplug($iface);
5015                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5016             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5017                 # Rate can be applied on its own but any change above needs to
5018                 # include the rate in tap_plug since OVS resets everything.
5019                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5020             }
5021
5022             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5023                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5024             }
5025
5026             return 1;
5027         }
5028     }
5029
5030     if ($hotplug) {
5031         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5032     } else {
5033         die "skip\n";
5034     }
5035 }
5036
5037 sub vmconfig_update_disk {
5038     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5039
5040     # fixme: do we need force?
5041
5042     my $drive = parse_drive($opt, $value);
5043
5044     if ($conf->{$opt}) {
5045
5046         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5047
5048             my $media = $drive->{media} || 'disk';
5049             my $oldmedia = $old_drive->{media} || 'disk';
5050             die "unable to change media type\n" if $media ne $oldmedia;
5051
5052             if (!drive_is_cdrom($old_drive)) {
5053
5054                 if ($drive->{file} ne $old_drive->{file}) {
5055
5056                     die "skip\n" if !$hotplug;
5057
5058                     # unplug and register as unused
5059                     vm_deviceunplug($vmid, $conf, $opt);
5060                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5061
5062                 } else {
5063                     # update existing disk
5064
5065                     # skip non hotpluggable value
5066                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5067                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5068                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5069                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5070                         die "skip\n";
5071                     }
5072
5073                     # apply throttle
5074                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5075                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5076                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5077                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5078                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5079                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5080                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5081                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5082                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5083                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5084                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5085                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5086                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5087                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5088                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5089                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5090                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5091                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5092
5093                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5094                                                    ($drive->{mbps} || 0)*1024*1024,
5095                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5096                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5097                                                    $drive->{iops} || 0,
5098                                                    $drive->{iops_rd} || 0,
5099                                                    $drive->{iops_wr} || 0,
5100                                                    ($drive->{mbps_max} || 0)*1024*1024,
5101                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5102                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5103                                                    $drive->{iops_max} || 0,
5104                                                    $drive->{iops_rd_max} || 0,
5105                                                    $drive->{iops_wr_max} || 0,
5106                                                    $drive->{bps_max_length} || 1,
5107                                                    $drive->{bps_rd_max_length} || 1,
5108                                                    $drive->{bps_wr_max_length} || 1,
5109                                                    $drive->{iops_max_length} || 1,
5110                                                    $drive->{iops_rd_max_length} || 1,
5111                                                    $drive->{iops_wr_max_length} || 1);
5112
5113                     }
5114
5115                     return 1;
5116                 }
5117
5118             } else { # cdrom
5119
5120                 if ($drive->{file} eq 'none') {
5121                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5122                     if (drive_is_cloudinit($old_drive)) {
5123                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5124                     }
5125                 } else {
5126                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5127                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5128                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5129                 }
5130
5131                 return 1;
5132             }
5133         }
5134     }
5135
5136     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5137     # hotplug new disks
5138     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5139     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5140 }
5141
5142 sub vm_start {
5143     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5144         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5145
5146     PVE::QemuConfig->lock_config($vmid, sub {
5147         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5148
5149         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5150
5151         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5152
5153         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5154
5155         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5156             vmconfig_apply_pending($vmid, $conf, $storecfg);
5157             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5158         }
5159
5160         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5161
5162         my $defaults = load_defaults();
5163
5164         # set environment variable useful inside network script
5165         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5166
5167         my $local_volumes = {};
5168
5169         if ($targetstorage) {
5170             foreach_drive($conf, sub {
5171                 my ($ds, $drive) = @_;
5172
5173                 return if drive_is_cdrom($drive);
5174
5175                 my $volid = $drive->{file};
5176
5177                 return if !$volid;
5178
5179                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5180
5181                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5182                 return if $scfg->{shared};
5183                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5184             });
5185
5186             my $format = undef;
5187
5188             foreach my $opt (sort keys %$local_volumes) {
5189
5190                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5191                 my $drive = parse_drive($opt, $conf->{$opt});
5192
5193                 #if remote storage is specified, use default format
5194                 if ($targetstorage && $targetstorage ne "1") {
5195                     $storeid = $targetstorage;
5196                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5197                     $format = $defFormat;
5198                 } else {
5199                     #else we use same format than original
5200                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5201                     $format = qemu_img_format($scfg, $volid);
5202                 }
5203
5204                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5205                 my $newdrive = $drive;
5206                 $newdrive->{format} = $format;
5207                 $newdrive->{file} = $newvolid;
5208                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5209                 $local_volumes->{$opt} = $drivestr;
5210                 #pass drive to conf for command line
5211                 $conf->{$opt} = $drivestr;
5212             }
5213         }
5214
5215         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5216
5217         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5218
5219         my $migrate_port = 0;
5220         my $migrate_uri;
5221         if ($statefile) {
5222             if ($statefile eq 'tcp') {
5223                 my $localip = "localhost";
5224                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5225                 my $nodename = PVE::INotify::nodename();
5226
5227                 if (!defined($migration_type)) {
5228                     if (defined($datacenterconf->{migration}->{type})) {
5229                         $migration_type = $datacenterconf->{migration}->{type};
5230                     } else {
5231                         $migration_type = 'secure';
5232                     }
5233                 }
5234
5235                 if ($migration_type eq 'insecure') {
5236                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5237                     if ($migrate_network_addr) {
5238                         $localip = $migrate_network_addr;
5239                     } else {
5240                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5241                     }
5242
5243                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5244                 }
5245
5246                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5247                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5248                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5249                 push @$cmd, '-incoming', $migrate_uri;
5250                 push @$cmd, '-S';
5251
5252             } elsif ($statefile eq 'unix') {
5253                 # should be default for secure migrations as a ssh TCP forward
5254                 # tunnel is not deterministic reliable ready and fails regurarly
5255                 # to set up in time, so use UNIX socket forwards
5256                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5257                 unlink $socket_addr;
5258
5259                 $migrate_uri = "unix:$socket_addr";
5260
5261                 push @$cmd, '-incoming', $migrate_uri;
5262                 push @$cmd, '-S';
5263
5264             } else {
5265                 push @$cmd, '-loadstate', $statefile;
5266             }
5267         } elsif ($paused) {
5268             push @$cmd, '-S';
5269         }
5270
5271         # host pci devices
5272         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5273           my $d = parse_hostpci($conf->{"hostpci$i"});
5274           next if !$d;
5275           my $pcidevices = $d->{pciid};
5276           foreach my $pcidevice (@$pcidevices) {
5277                 my $pciid = $pcidevice->{id};
5278
5279                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5280                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5281                 die "no pci device info for device '$pciid'\n" if !$info;
5282
5283                 if ($d->{mdev}) {
5284                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5285                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5286                 } else {
5287                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5288                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5289                     die "can't reset pci device '$pciid'\n"
5290                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5291                 }
5292           }
5293         }
5294
5295         PVE::Storage::activate_volumes($storecfg, $vollist);
5296
5297         if (-d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
5298             eval {
5299                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5300                     outfunc => sub {}, errfunc => sub {});
5301             };
5302         }
5303
5304         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5305                                                   : $defaults->{cpuunits};
5306
5307         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5308         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5309
5310         my %properties = (
5311             Slice => 'qemu.slice',
5312             KillMode => 'none',
5313             CPUShares => $cpuunits
5314         );
5315
5316         if (my $cpulimit = $conf->{cpulimit}) {
5317             $properties{CPUQuota} = int($cpulimit * 100);
5318         }
5319         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5320
5321         my $run_qemu = sub {
5322             PVE::Tools::run_fork sub {
5323                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5324                 run_command($cmd, %run_params);
5325             };
5326         };
5327
5328         if ($conf->{hugepages}) {
5329
5330             my $code = sub {
5331                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5332                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5333
5334                 PVE::QemuServer::Memory::hugepages_mount();
5335                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5336
5337                 eval { $run_qemu->() };
5338                 if (my $err = $@) {
5339                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5340                     die $err;
5341                 }
5342
5343                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5344             };
5345             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5346
5347         } else {
5348             eval { $run_qemu->() };
5349         }
5350
5351         if (my $err = $@) {
5352             # deactivate volumes if start fails
5353             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5354             die "start failed: $err";
5355         }
5356
5357         print "migration listens on $migrate_uri\n" if $migrate_uri;
5358
5359         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5360             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5361             warn $@ if $@;
5362         }
5363
5364         #start nbd server for storage migration
5365         if ($targetstorage) {
5366             my $nodename = PVE::INotify::nodename();
5367             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5368             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5369             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5370             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5371
5372             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5373
5374             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5375
5376             foreach my $opt (sort keys %$local_volumes) {
5377                 my $volid = $local_volumes->{$opt};
5378                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5379                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5380                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5381             }
5382         }
5383
5384         if ($migratedfrom) {
5385             eval {
5386                 set_migration_caps($vmid);
5387             };
5388             warn $@ if $@;
5389
5390             if ($spice_port) {
5391                 print "spice listens on port $spice_port\n";
5392                 if ($spice_ticket) {
5393                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5394                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5395                 }
5396             }
5397
5398         } else {
5399             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5400                 if !$statefile && $conf->{balloon};
5401
5402             foreach my $opt (keys %$conf) {
5403                 next if $opt !~  m/^net\d+$/;
5404                 my $nicconf = parse_net($conf->{$opt});
5405                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5406             }
5407         }
5408
5409         vm_mon_cmd_nocheck($vmid, 'qom-set',
5410                     path => "machine/peripheral/balloon0",
5411                     property => "guest-stats-polling-interval",
5412                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5413
5414         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5415     });
5416 }
5417
5418 sub vm_mon_cmd {
5419     my ($vmid, $execute, %params) = @_;
5420
5421     my $cmd = { execute => $execute, arguments => \%params };
5422     vm_qmp_command($vmid, $cmd);
5423 }
5424
5425 sub vm_mon_cmd_nocheck {
5426     my ($vmid, $execute, %params) = @_;
5427
5428     my $cmd = { execute => $execute, arguments => \%params };
5429     vm_qmp_command($vmid, $cmd, 1);
5430 }
5431
5432 sub vm_qmp_command {
5433     my ($vmid, $cmd, $nocheck) = @_;
5434
5435     my $res;
5436
5437     my $timeout;
5438     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5439         $timeout = $cmd->{arguments}->{timeout};
5440         delete $cmd->{arguments}->{timeout};
5441     }
5442
5443     eval {
5444         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5445         my $sname = qmp_socket($vmid);
5446         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5447             my $qmpclient = PVE::QMPClient->new();
5448
5449             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5450         } else {
5451             die "unable to open monitor socket\n";
5452         }
5453     };
5454     if (my $err = $@) {
5455         syslog("err", "VM $vmid qmp command failed - $err");
5456         die $err;
5457     }
5458
5459     return $res;
5460 }
5461
5462 sub vm_human_monitor_command {
5463     my ($vmid, $cmdline) = @_;
5464
5465     my $res;
5466
5467     my $cmd = {
5468         execute => 'human-monitor-command',
5469         arguments => { 'command-line' => $cmdline},
5470     };
5471
5472     return vm_qmp_command($vmid, $cmd);
5473 }
5474
5475 sub vm_commandline {
5476     my ($storecfg, $vmid, $snapname) = @_;
5477
5478     my $conf = PVE::QemuConfig->load_config($vmid);
5479
5480     if ($snapname) {
5481         my $snapshot = $conf->{snapshots}->{$snapname};
5482         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5483
5484         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5485
5486         $conf = $snapshot;
5487     }
5488
5489     my $defaults = load_defaults();
5490
5491     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5492
5493     return PVE::Tools::cmd2string($cmd);
5494 }
5495
5496 sub vm_reset {
5497     my ($vmid, $skiplock) = @_;
5498
5499     PVE::QemuConfig->lock_config($vmid, sub {
5500
5501         my $conf = PVE::QemuConfig->load_config($vmid);
5502
5503         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5504
5505         vm_mon_cmd($vmid, "system_reset");
5506     });
5507 }
5508
5509 sub get_vm_volumes {
5510     my ($conf) = @_;
5511
5512     my $vollist = [];
5513     foreach_volid($conf, sub {
5514         my ($volid, $attr) = @_;
5515
5516         return if $volid =~ m|^/|;
5517
5518         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5519         return if !$sid;
5520
5521         push @$vollist, $volid;
5522     });
5523
5524     return $vollist;
5525 }
5526
5527 sub vm_stop_cleanup {
5528     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5529
5530     eval {
5531
5532         if (!$keepActive) {
5533             my $vollist = get_vm_volumes($conf);
5534             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5535         }
5536
5537         foreach my $ext (qw(mon qmp pid vnc qga)) {
5538             unlink "/var/run/qemu-server/${vmid}.$ext";
5539         }
5540
5541         if ($conf->{ivshmem}) {
5542             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5543             # just delete it for now, VMs which have this already open do not
5544             # are affected, but new VMs will get a separated one. If this
5545             # becomes an issue we either add some sort of ref-counting or just
5546             # add a "don't delete on stop" flag to the ivshmem format.
5547             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5548         }
5549
5550         foreach my $key (keys %$conf) {
5551             next if $key !~ m/^hostpci(\d+)$/;
5552             my $hostpciindex = $1;
5553             my $d = parse_hostpci($conf->{$key});
5554             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5555
5556             foreach my $pci (@{$d->{pciid}}) {
5557                 my $pciid = $pci->{id};
5558                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5559             }
5560         }
5561
5562         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5563     };
5564     warn $@ if $@; # avoid errors - just warn
5565 }
5566
5567 # Note: use $nockeck to skip tests if VM configuration file exists.
5568 # We need that when migration VMs to other nodes (files already moved)
5569 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5570 sub vm_stop {
5571     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5572
5573     $force = 1 if !defined($force) && !$shutdown;
5574
5575     if ($migratedfrom){
5576         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5577         kill 15, $pid if $pid;
5578         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5579         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5580         return;
5581     }
5582
5583     PVE::QemuConfig->lock_config($vmid, sub {
5584
5585         my $pid = check_running($vmid, $nocheck);
5586         return if !$pid;
5587
5588         my $conf;
5589         if (!$nocheck) {
5590             $conf = PVE::QemuConfig->load_config($vmid);
5591             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5592             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5593                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5594                 $timeout = $opts->{down} if $opts->{down};
5595             }
5596             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5597         }
5598
5599         $timeout = 60 if !defined($timeout);
5600
5601         eval {
5602             if ($shutdown) {
5603                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5604                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5605                 } else {
5606                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5607                 }
5608             } else {
5609                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5610             }
5611         };
5612         my $err = $@;
5613
5614         if (!$err) {
5615             my $count = 0;
5616             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5617                 $count++;
5618                 sleep 1;
5619             }
5620
5621             if ($count >= $timeout) {
5622                 if ($force) {
5623                     warn "VM still running - terminating now with SIGTERM\n";
5624                     kill 15, $pid;
5625                 } else {
5626                     die "VM quit/powerdown failed - got timeout\n";
5627                 }
5628             } else {
5629                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5630                 return;
5631             }
5632         } else {
5633             if ($force) {
5634                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5635                 kill 15, $pid;
5636             } else {
5637                 die "VM quit/powerdown failed\n";
5638             }
5639         }
5640
5641         # wait again
5642         $timeout = 10;
5643
5644         my $count = 0;
5645         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5646             $count++;
5647             sleep 1;
5648         }
5649
5650         if ($count >= $timeout) {
5651             warn "VM still running - terminating now with SIGKILL\n";
5652             kill 9, $pid;
5653             sleep 1;
5654         }
5655
5656         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5657    });
5658 }
5659
5660 sub vm_suspend {
5661     my ($vmid, $skiplock, $includestate) = @_;
5662
5663     my $conf;
5664     my $path;
5665     my $storecfg;
5666     my $vmstate;
5667
5668     PVE::QemuConfig->lock_config($vmid, sub {
5669
5670         $conf = PVE::QemuConfig->load_config($vmid);
5671
5672         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5673         PVE::QemuConfig->check_lock($conf)
5674             if !($skiplock || $is_backing_up);
5675
5676         die "cannot suspend to disk during backup\n"
5677             if $is_backing_up && $includestate;
5678
5679         if ($includestate) {
5680             $conf->{lock} = 'suspending';
5681             my $date = strftime("%Y-%m-%d", localtime(time()));
5682             $storecfg = PVE::Storage::config();
5683             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, 1);
5684             $path = PVE::Storage::path($storecfg, $vmstate);
5685             PVE::QemuConfig->write_config($vmid, $conf);
5686         } else {
5687             vm_mon_cmd($vmid, "stop");
5688         }
5689     });
5690
5691     if ($includestate) {
5692         # save vm state
5693         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5694
5695         eval {
5696             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5697             for(;;) {
5698                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5699                 if (!$state->{status}) {
5700                     die "savevm not active\n";
5701                 } elsif ($state->{status} eq 'active') {
5702                     sleep(1);
5703                     next;
5704                 } elsif ($state->{status} eq 'completed') {
5705                     last;
5706                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5707                     die "query-savevm failed with error '$state->{error}'\n"
5708                 } else {
5709                     die "query-savevm returned status '$state->{status}'\n";
5710                 }
5711             }
5712         };
5713         my $err = $@;
5714
5715         PVE::QemuConfig->lock_config($vmid, sub {
5716             $conf = PVE::QemuConfig->load_config($vmid);
5717             if ($err) {
5718                 # cleanup, but leave suspending lock, to indicate something went wrong
5719                 eval {
5720                     vm_mon_cmd($vmid, "savevm-end");
5721                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5722                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5723                     delete $conf->@{qw(vmstate runningmachine)};
5724                     PVE::QemuConfig->write_config($vmid, $conf);
5725                 };
5726                 warn $@ if $@;
5727                 die $err;
5728             }
5729
5730             die "lock changed unexpectedly\n"
5731                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5732
5733             vm_qmp_command($vmid, { execute => "quit" });
5734             $conf->{lock} = 'suspended';
5735             PVE::QemuConfig->write_config($vmid, $conf);
5736         });
5737     }
5738 }
5739
5740 sub vm_resume {
5741     my ($vmid, $skiplock, $nocheck) = @_;
5742
5743     PVE::QemuConfig->lock_config($vmid, sub {
5744
5745         my $res = vm_mon_cmd($vmid, 'query-status');
5746         my $resume_cmd = 'cont';
5747
5748         if ($res->{status} && $res->{status} eq 'suspended') {
5749             $resume_cmd = 'system_wakeup';
5750         }
5751
5752         if (!$nocheck) {
5753
5754             my $conf = PVE::QemuConfig->load_config($vmid);
5755
5756             PVE::QemuConfig->check_lock($conf)
5757                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5758
5759             vm_mon_cmd($vmid, $resume_cmd);
5760
5761         } else {
5762             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5763         }
5764     });
5765 }
5766
5767 sub vm_sendkey {
5768     my ($vmid, $skiplock, $key) = @_;
5769
5770     PVE::QemuConfig->lock_config($vmid, sub {
5771
5772         my $conf = PVE::QemuConfig->load_config($vmid);
5773
5774         # there is no qmp command, so we use the human monitor command
5775         vm_human_monitor_command($vmid, "sendkey $key");
5776     });
5777 }
5778
5779 sub vm_destroy {
5780     my ($storecfg, $vmid, $skiplock) = @_;
5781
5782     PVE::QemuConfig->lock_config($vmid, sub {
5783
5784         my $conf = PVE::QemuConfig->load_config($vmid);
5785
5786         if (!check_running($vmid)) {
5787             destroy_vm($storecfg, $vmid, undef, $skiplock);
5788         } else {
5789             die "VM $vmid is running - destroy failed\n";
5790         }
5791     });
5792 }
5793
5794 # vzdump restore implementaion
5795
5796 sub tar_archive_read_firstfile {
5797     my $archive = shift;
5798
5799     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5800
5801     # try to detect archive type first
5802     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5803         die "unable to open file '$archive'\n";
5804     my $firstfile = <$fh>;
5805     kill 15, $pid;
5806     close $fh;
5807
5808     die "ERROR: archive contaions no data\n" if !$firstfile;
5809     chomp $firstfile;
5810
5811     return $firstfile;
5812 }
5813
5814 sub tar_restore_cleanup {
5815     my ($storecfg, $statfile) = @_;
5816
5817     print STDERR "starting cleanup\n";
5818
5819     if (my $fd = IO::File->new($statfile, "r")) {
5820         while (defined(my $line = <$fd>)) {
5821             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5822                 my $volid = $2;
5823                 eval {
5824                     if ($volid =~ m|^/|) {
5825                         unlink $volid || die 'unlink failed\n';
5826                     } else {
5827                         PVE::Storage::vdisk_free($storecfg, $volid);
5828                     }
5829                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5830                 };
5831                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5832             } else {
5833                 print STDERR "unable to parse line in statfile - $line";
5834             }
5835         }
5836         $fd->close();
5837     }
5838 }
5839
5840 sub restore_archive {
5841     my ($archive, $vmid, $user, $opts) = @_;
5842
5843     my $format = $opts->{format};
5844     my $comp;
5845
5846     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5847         $format = 'tar' if !$format;
5848         $comp = 'gzip';
5849     } elsif ($archive =~ m/\.tar$/) {
5850         $format = 'tar' if !$format;
5851     } elsif ($archive =~ m/.tar.lzo$/) {
5852         $format = 'tar' if !$format;
5853         $comp = 'lzop';
5854     } elsif ($archive =~ m/\.vma$/) {
5855         $format = 'vma' if !$format;
5856     } elsif ($archive =~ m/\.vma\.gz$/) {
5857         $format = 'vma' if !$format;
5858         $comp = 'gzip';
5859     } elsif ($archive =~ m/\.vma\.lzo$/) {
5860         $format = 'vma' if !$format;
5861         $comp = 'lzop';
5862     } else {
5863         $format = 'vma' if !$format; # default
5864     }
5865
5866     # try to detect archive format
5867     if ($format eq 'tar') {
5868         return restore_tar_archive($archive, $vmid, $user, $opts);
5869     } else {
5870         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5871     }
5872 }
5873
5874 sub restore_update_config_line {
5875     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5876
5877     return if $line =~ m/^\#qmdump\#/;
5878     return if $line =~ m/^\#vzdump\#/;
5879     return if $line =~ m/^lock:/;
5880     return if $line =~ m/^unused\d+:/;
5881     return if $line =~ m/^parent:/;
5882     return if $line =~ m/^template:/; # restored VM is never a template
5883
5884     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5885     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5886         # try to convert old 1.X settings
5887         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5888         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5889             my ($model, $macaddr) = split(/\=/, $devconfig);
5890             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5891             my $net = {
5892                 model => $model,
5893                 bridge => "vmbr$ind",
5894                 macaddr => $macaddr,
5895             };
5896             my $netstr = print_net($net);
5897
5898             print $outfd "net$cookie->{netcount}: $netstr\n";
5899             $cookie->{netcount}++;
5900         }
5901     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5902         my ($id, $netstr) = ($1, $2);
5903         my $net = parse_net($netstr);
5904         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5905         $netstr = print_net($net);
5906         print $outfd "$id: $netstr\n";
5907     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5908         my $virtdev = $1;
5909         my $value = $3;
5910         my $di = parse_drive($virtdev, $value);
5911         if (defined($di->{backup}) && !$di->{backup}) {
5912             print $outfd "#$line";
5913         } elsif ($map->{$virtdev}) {
5914             delete $di->{format}; # format can change on restore
5915             $di->{file} = $map->{$virtdev};
5916             $value = print_drive($vmid, $di);
5917             print $outfd "$virtdev: $value\n";
5918         } else {
5919             print $outfd $line;
5920         }
5921     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5922         my $vmgenid = $1;
5923         if ($vmgenid ne '0') {
5924             # always generate a new vmgenid if there was a valid one setup
5925             $vmgenid = generate_uuid();
5926         }
5927         print $outfd "vmgenid: $vmgenid\n";
5928     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5929         my ($uuid, $uuid_str);
5930         UUID::generate($uuid);
5931         UUID::unparse($uuid, $uuid_str);
5932         my $smbios1 = parse_smbios1($2);
5933         $smbios1->{uuid} = $uuid_str;
5934         print $outfd $1.print_smbios1($smbios1)."\n";
5935     } else {
5936         print $outfd $line;
5937     }
5938 }
5939
5940 sub scan_volids {
5941     my ($cfg, $vmid) = @_;
5942
5943     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5944
5945     my $volid_hash = {};
5946     foreach my $storeid (keys %$info) {
5947         foreach my $item (@{$info->{$storeid}}) {
5948             next if !($item->{volid} && $item->{size});
5949             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5950             $volid_hash->{$item->{volid}} = $item;
5951         }
5952     }
5953
5954     return $volid_hash;
5955 }
5956
5957 sub is_volume_in_use {
5958     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5959
5960     my $path = PVE::Storage::path($storecfg, $volid);
5961
5962     my $scan_config = sub {
5963         my ($cref, $snapname) = @_;
5964
5965         foreach my $key (keys %$cref) {
5966             my $value = $cref->{$key};
5967             if (is_valid_drivename($key)) {
5968                 next if $skip_drive && $key eq $skip_drive;
5969                 my $drive = parse_drive($key, $value);
5970                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5971                 return 1 if $volid eq $drive->{file};
5972                 if ($drive->{file} =~ m!^/!) {
5973                     return 1 if $drive->{file} eq $path;
5974                 } else {
5975                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5976                     next if !$storeid;
5977                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5978                     next if !$scfg;
5979                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5980                 }
5981             }
5982         }
5983
5984         return 0;
5985     };
5986
5987     return 1 if &$scan_config($conf);
5988
5989     undef $skip_drive;
5990
5991     foreach my $snapname (keys %{$conf->{snapshots}}) {
5992         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5993     }
5994
5995     return 0;
5996 }
5997
5998 sub update_disksize {
5999     my ($vmid, $conf, $volid_hash) = @_;
6000
6001     my $changes;
6002     my $prefix = "VM $vmid:";
6003
6004     # used and unused disks
6005     my $referenced = {};
6006
6007     # Note: it is allowed to define multiple storages with same path (alias), so
6008     # we need to check both 'volid' and real 'path' (two different volid can point
6009     # to the same path).
6010
6011     my $referencedpath = {};
6012
6013     # update size info
6014     foreach my $opt (keys %$conf) {
6015         if (is_valid_drivename($opt)) {
6016             my $drive = parse_drive($opt, $conf->{$opt});
6017             my $volid = $drive->{file};
6018             next if !$volid;
6019
6020             $referenced->{$volid} = 1;
6021             if ($volid_hash->{$volid} &&
6022                 (my $path = $volid_hash->{$volid}->{path})) {
6023                 $referencedpath->{$path} = 1;
6024             }
6025
6026             next if drive_is_cdrom($drive);
6027             next if !$volid_hash->{$volid};
6028
6029             $drive->{size} = $volid_hash->{$volid}->{size};
6030             my $new = print_drive($vmid, $drive);
6031             if ($new ne $conf->{$opt}) {
6032                 $changes = 1;
6033                 $conf->{$opt} = $new;
6034                 print "$prefix update disk '$opt' information.\n";
6035             }
6036         }
6037     }
6038
6039     # remove 'unusedX' entry if volume is used
6040     foreach my $opt (keys %$conf) {
6041         next if $opt !~ m/^unused\d+$/;
6042         my $volid = $conf->{$opt};
6043         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6044         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6045             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6046             $changes = 1;
6047             delete $conf->{$opt};
6048         }
6049
6050         $referenced->{$volid} = 1;
6051         $referencedpath->{$path} = 1 if $path;
6052     }
6053
6054     foreach my $volid (sort keys %$volid_hash) {
6055         next if $volid =~ m/vm-$vmid-state-/;
6056         next if $referenced->{$volid};
6057         my $path = $volid_hash->{$volid}->{path};
6058         next if !$path; # just to be sure
6059         next if $referencedpath->{$path};
6060         $changes = 1;
6061         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6062         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6063         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6064     }
6065
6066     return $changes;
6067 }
6068
6069 sub rescan {
6070     my ($vmid, $nolock, $dryrun) = @_;
6071
6072     my $cfg = PVE::Storage::config();
6073
6074     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6075     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6076     foreach my $stor (keys %{$cfg->{ids}}) {
6077         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6078     }
6079
6080     print "rescan volumes...\n";
6081     my $volid_hash = scan_volids($cfg, $vmid);
6082
6083     my $updatefn =  sub {
6084         my ($vmid) = @_;
6085
6086         my $conf = PVE::QemuConfig->load_config($vmid);
6087
6088         PVE::QemuConfig->check_lock($conf);
6089
6090         my $vm_volids = {};
6091         foreach my $volid (keys %$volid_hash) {
6092             my $info = $volid_hash->{$volid};
6093             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6094         }
6095
6096         my $changes = update_disksize($vmid, $conf, $vm_volids);
6097
6098         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6099     };
6100
6101     if (defined($vmid)) {
6102         if ($nolock) {
6103             &$updatefn($vmid);
6104         } else {
6105             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6106         }
6107     } else {
6108         my $vmlist = config_list();
6109         foreach my $vmid (keys %$vmlist) {
6110             if ($nolock) {
6111                 &$updatefn($vmid);
6112             } else {
6113                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6114             }
6115         }
6116     }
6117 }
6118
6119 sub restore_vma_archive {
6120     my ($archive, $vmid, $user, $opts, $comp) = @_;
6121
6122     my $readfrom = $archive;
6123
6124     my $cfg = PVE::Storage::config();
6125     my $commands = [];
6126     my $bwlimit = $opts->{bwlimit};
6127
6128     my $dbg_cmdstring = '';
6129     my $add_pipe = sub {
6130         my ($cmd) = @_;
6131         push @$commands, $cmd;
6132         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6133         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6134         $readfrom = '-';
6135     };
6136
6137     my $input = undef;
6138     if ($archive eq '-') {
6139         $input = '<&STDIN';
6140     } else {
6141         # If we use a backup from a PVE defined storage we also consider that
6142         # storage's rate limit:
6143         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6144         if (defined($volid)) {
6145             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6146             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6147             if ($readlimit) {
6148                 print STDERR "applying read rate limit: $readlimit\n";
6149                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6150                 $add_pipe->($cstream);
6151             }
6152         }
6153     }
6154
6155     if ($comp) {
6156         my $cmd;
6157         if ($comp eq 'gzip') {
6158             $cmd = ['zcat', $readfrom];
6159         } elsif ($comp eq 'lzop') {
6160             $cmd = ['lzop', '-d', '-c', $readfrom];
6161         } else {
6162             die "unknown compression method '$comp'\n";
6163         }
6164         $add_pipe->($cmd);
6165     }
6166
6167     my $tmpdir = "/var/tmp/vzdumptmp$$";
6168     rmtree $tmpdir;
6169
6170     # disable interrupts (always do cleanups)
6171     local $SIG{INT} =
6172         local $SIG{TERM} =
6173         local $SIG{QUIT} =
6174         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6175
6176     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6177     POSIX::mkfifo($mapfifo, 0600);
6178     my $fifofh;
6179
6180     my $openfifo = sub {
6181         open($fifofh, '>', $mapfifo) || die $!;
6182     };
6183
6184     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6185
6186     my $oldtimeout;
6187     my $timeout = 5;
6188
6189     my $devinfo = {};
6190
6191     my $rpcenv = PVE::RPCEnvironment::get();
6192
6193     my $conffile = PVE::QemuConfig->config_file($vmid);
6194     my $tmpfn = "$conffile.$$.tmp";
6195
6196     # Note: $oldconf is undef if VM does not exists
6197     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6198     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6199
6200     my %storage_limits;
6201
6202     my $print_devmap = sub {
6203         my $virtdev_hash = {};
6204
6205         my $cfgfn = "$tmpdir/qemu-server.conf";
6206
6207         # we can read the config - that is already extracted
6208         my $fh = IO::File->new($cfgfn, "r") ||
6209             "unable to read qemu-server.conf - $!\n";
6210
6211         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6212         if (-f $fwcfgfn) {
6213             my $pve_firewall_dir = '/etc/pve/firewall';
6214             mkdir $pve_firewall_dir; # make sure the dir exists
6215             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6216         }
6217
6218         while (defined(my $line = <$fh>)) {
6219             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6220                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6221                 die "archive does not contain data for drive '$virtdev'\n"
6222                     if !$devinfo->{$devname};
6223                 if (defined($opts->{storage})) {
6224                     $storeid = $opts->{storage} || 'local';
6225                 } elsif (!$storeid) {
6226                     $storeid = 'local';
6227                 }
6228                 $format = 'raw' if !$format;
6229                 $devinfo->{$devname}->{devname} = $devname;
6230                 $devinfo->{$devname}->{virtdev} = $virtdev;
6231                 $devinfo->{$devname}->{format} = $format;
6232                 $devinfo->{$devname}->{storeid} = $storeid;
6233
6234                 # check permission on storage
6235                 my $pool = $opts->{pool}; # todo: do we need that?
6236                 if ($user ne 'root@pam') {
6237                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6238                 }
6239
6240                 $storage_limits{$storeid} = $bwlimit;
6241
6242                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6243             }
6244         }
6245
6246         foreach my $key (keys %storage_limits) {
6247             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6248             next if !$limit;
6249             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6250             $storage_limits{$key} = $limit * 1024;
6251         }
6252
6253         foreach my $devname (keys %$devinfo) {
6254             die "found no device mapping information for device '$devname'\n"
6255                 if !$devinfo->{$devname}->{virtdev};
6256         }
6257
6258         # create empty/temp config
6259         if ($oldconf) {
6260             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6261             foreach_drive($oldconf, sub {
6262                 my ($ds, $drive) = @_;
6263
6264                 return if drive_is_cdrom($drive);
6265
6266                 my $volid = $drive->{file};
6267
6268                 return if !$volid || $volid =~ m|^/|;
6269
6270                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6271                 return if !$path || !$owner || ($owner != $vmid);
6272
6273                 # Note: only delete disk we want to restore
6274                 # other volumes will become unused
6275                 if ($virtdev_hash->{$ds}) {
6276                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6277                     if (my $err = $@) {
6278                         warn $err;
6279                     }
6280                 }
6281             });
6282
6283             # delete vmstate files
6284             # since after the restore we have no snapshots anymore
6285             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6286                 my $snap = $oldconf->{snapshots}->{$snapname};
6287                 if ($snap->{vmstate}) {
6288                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6289                     if (my $err = $@) {
6290                         warn $err;
6291                     }
6292                 }
6293             }
6294         }
6295
6296         my $map = {};
6297         foreach my $virtdev (sort keys %$virtdev_hash) {
6298             my $d = $virtdev_hash->{$virtdev};
6299             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6300             my $storeid = $d->{storeid};
6301             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6302
6303             my $map_opts = '';
6304             if (my $limit = $storage_limits{$storeid}) {
6305                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6306             }
6307
6308             # test if requested format is supported
6309             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6310             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6311             $d->{format} = $defFormat if !$supported;
6312
6313             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6314                                                   $d->{format}, undef, $alloc_size);
6315             print STDERR "new volume ID is '$volid'\n";
6316             $d->{volid} = $volid;
6317             my $path = PVE::Storage::path($cfg, $volid);
6318
6319             PVE::Storage::activate_volumes($cfg,[$volid]);
6320
6321             my $write_zeros = 1;
6322             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6323                 $write_zeros = 0;
6324             }
6325
6326             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6327
6328             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6329             $map->{$virtdev} = $volid;
6330         }
6331
6332         $fh->seek(0, 0) || die "seek failed - $!\n";
6333
6334         my $outfd = new IO::File ($tmpfn, "w") ||
6335             die "unable to write config for VM $vmid\n";
6336
6337         my $cookie = { netcount => 0 };
6338         while (defined(my $line = <$fh>)) {
6339             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6340         }
6341
6342         $fh->close();
6343         $outfd->close();
6344     };
6345
6346     eval {
6347         # enable interrupts
6348         local $SIG{INT} =
6349             local $SIG{TERM} =
6350             local $SIG{QUIT} =
6351             local $SIG{HUP} =
6352             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6353         local $SIG{ALRM} = sub { die "got timeout\n"; };
6354
6355         $oldtimeout = alarm($timeout);
6356
6357         my $parser = sub {
6358             my $line = shift;
6359
6360             print "$line\n";
6361
6362             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6363                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6364                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6365             } elsif ($line =~ m/^CTIME: /) {
6366                 # we correctly received the vma config, so we can disable
6367                 # the timeout now for disk allocation (set to 10 minutes, so
6368                 # that we always timeout if something goes wrong)
6369                 alarm(600);
6370                 &$print_devmap();
6371                 print $fifofh "done\n";
6372                 my $tmp = $oldtimeout || 0;
6373                 $oldtimeout = undef;
6374                 alarm($tmp);
6375                 close($fifofh);
6376             }
6377         };
6378
6379         print "restore vma archive: $dbg_cmdstring\n";
6380         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6381     };
6382     my $err = $@;
6383
6384     alarm($oldtimeout) if $oldtimeout;
6385
6386     my $vollist = [];
6387     foreach my $devname (keys %$devinfo) {
6388         my $volid = $devinfo->{$devname}->{volid};
6389         push @$vollist, $volid if $volid;
6390     }
6391
6392     PVE::Storage::deactivate_volumes($cfg, $vollist);
6393
6394     unlink $mapfifo;
6395
6396     if ($err) {
6397         rmtree $tmpdir;
6398         unlink $tmpfn;
6399
6400         foreach my $devname (keys %$devinfo) {
6401             my $volid = $devinfo->{$devname}->{volid};
6402             next if !$volid;
6403             eval {
6404                 if ($volid =~ m|^/|) {
6405                     unlink $volid || die 'unlink failed\n';
6406                 } else {
6407                     PVE::Storage::vdisk_free($cfg, $volid);
6408                 }
6409                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6410             };
6411             print STDERR "unable to cleanup '$volid' - $@" if $@;
6412         }
6413         die $err;
6414     }
6415
6416     rmtree $tmpdir;
6417
6418     rename($tmpfn, $conffile) ||
6419         die "unable to commit configuration file '$conffile'\n";
6420
6421     PVE::Cluster::cfs_update(); # make sure we read new file
6422
6423     eval { rescan($vmid, 1); };
6424     warn $@ if $@;
6425 }
6426
6427 sub restore_tar_archive {
6428     my ($archive, $vmid, $user, $opts) = @_;
6429
6430     if ($archive ne '-') {
6431         my $firstfile = tar_archive_read_firstfile($archive);
6432         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6433             if $firstfile ne 'qemu-server.conf';
6434     }
6435
6436     my $storecfg = PVE::Storage::config();
6437
6438     # destroy existing data - keep empty config
6439     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6440     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6441
6442     my $tocmd = "/usr/lib/qemu-server/qmextract";
6443
6444     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6445     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6446     $tocmd .= ' --prealloc' if $opts->{prealloc};
6447     $tocmd .= ' --info' if $opts->{info};
6448
6449     # tar option "xf" does not autodetect compression when read from STDIN,
6450     # so we pipe to zcat
6451     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6452         PVE::Tools::shellquote("--to-command=$tocmd");
6453
6454     my $tmpdir = "/var/tmp/vzdumptmp$$";
6455     mkpath $tmpdir;
6456
6457     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6458     local $ENV{VZDUMP_VMID} = $vmid;
6459     local $ENV{VZDUMP_USER} = $user;
6460
6461     my $conffile = PVE::QemuConfig->config_file($vmid);
6462     my $tmpfn = "$conffile.$$.tmp";
6463
6464     # disable interrupts (always do cleanups)
6465     local $SIG{INT} =
6466         local $SIG{TERM} =
6467         local $SIG{QUIT} =
6468         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6469
6470     eval {
6471         # enable interrupts
6472         local $SIG{INT} =
6473             local $SIG{TERM} =
6474             local $SIG{QUIT} =
6475             local $SIG{HUP} =
6476             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6477
6478         if ($archive eq '-') {
6479             print "extracting archive from STDIN\n";
6480             run_command($cmd, input => "<&STDIN");
6481         } else {
6482             print "extracting archive '$archive'\n";
6483             run_command($cmd);
6484         }
6485
6486         return if $opts->{info};
6487
6488         # read new mapping
6489         my $map = {};
6490         my $statfile = "$tmpdir/qmrestore.stat";
6491         if (my $fd = IO::File->new($statfile, "r")) {
6492             while (defined (my $line = <$fd>)) {
6493                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6494                     $map->{$1} = $2 if $1;
6495                 } else {
6496                     print STDERR "unable to parse line in statfile - $line\n";
6497                 }
6498             }
6499             $fd->close();
6500         }
6501
6502         my $confsrc = "$tmpdir/qemu-server.conf";
6503
6504         my $srcfd = new IO::File($confsrc, "r") ||
6505             die "unable to open file '$confsrc'\n";
6506
6507         my $outfd = new IO::File ($tmpfn, "w") ||
6508             die "unable to write config for VM $vmid\n";
6509
6510         my $cookie = { netcount => 0 };
6511         while (defined (my $line = <$srcfd>)) {
6512             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6513         }
6514
6515         $srcfd->close();
6516         $outfd->close();
6517     };
6518     my $err = $@;
6519
6520     if ($err) {
6521
6522         unlink $tmpfn;
6523
6524         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6525
6526         die $err;
6527     }
6528
6529     rmtree $tmpdir;
6530
6531     rename $tmpfn, $conffile ||
6532         die "unable to commit configuration file '$conffile'\n";
6533
6534     PVE::Cluster::cfs_update(); # make sure we read new file
6535
6536     eval { rescan($vmid, 1); };
6537     warn $@ if $@;
6538 };
6539
6540 sub foreach_storage_used_by_vm {
6541     my ($conf, $func) = @_;
6542
6543     my $sidhash = {};
6544
6545     foreach_drive($conf, sub {
6546         my ($ds, $drive) = @_;
6547         return if drive_is_cdrom($drive);
6548
6549         my $volid = $drive->{file};
6550
6551         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6552         $sidhash->{$sid} = $sid if $sid;
6553     });
6554
6555     foreach my $sid (sort keys %$sidhash) {
6556         &$func($sid);
6557     }
6558 }
6559
6560 sub do_snapshots_with_qemu {
6561     my ($storecfg, $volid) = @_;
6562
6563     my $storage_name = PVE::Storage::parse_volume_id($volid);
6564
6565     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6566         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6567         return 1;
6568     }
6569
6570     if ($volid =~ m/\.(qcow2|qed)$/){
6571         return 1;
6572     }
6573
6574     return undef;
6575 }
6576
6577 sub qga_check_running {
6578     my ($vmid, $nowarn) = @_;
6579
6580     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6581     if ($@) {
6582         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6583         return 0;
6584     }
6585     return 1;
6586 }
6587
6588 sub template_create {
6589     my ($vmid, $conf, $disk) = @_;
6590
6591     my $storecfg = PVE::Storage::config();
6592
6593     foreach_drive($conf, sub {
6594         my ($ds, $drive) = @_;
6595
6596         return if drive_is_cdrom($drive);
6597         return if $disk && $ds ne $disk;
6598
6599         my $volid = $drive->{file};
6600         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6601
6602         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6603         $drive->{file} = $voliddst;
6604         $conf->{$ds} = print_drive($vmid, $drive);
6605         PVE::QemuConfig->write_config($vmid, $conf);
6606     });
6607 }
6608
6609 sub convert_iscsi_path {
6610     my ($path) = @_;
6611
6612     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6613         my $portal = $1;
6614         my $target = $2;
6615         my $lun = $3;
6616
6617         my $initiator_name = get_initiator_name();
6618
6619         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6620                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6621     }
6622
6623     die "cannot convert iscsi path '$path', unkown format\n";
6624 }
6625
6626 sub qemu_img_convert {
6627     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6628
6629     my $storecfg = PVE::Storage::config();
6630     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6631     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6632
6633     if ($src_storeid && $dst_storeid) {
6634
6635         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6636
6637         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6638         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6639
6640         my $src_format = qemu_img_format($src_scfg, $src_volname);
6641         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6642
6643         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6644         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6645
6646         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6647         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6648
6649         my $cmd = [];
6650         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6651         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6652         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6653         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6654
6655         if ($src_is_iscsi) {
6656             push @$cmd, '--image-opts';
6657             $src_path = convert_iscsi_path($src_path);
6658         } else {
6659             push @$cmd, '-f', $src_format;
6660         }
6661
6662         if ($dst_is_iscsi) {
6663             push @$cmd, '--target-image-opts';
6664             $dst_path = convert_iscsi_path($dst_path);
6665         } else {
6666             push @$cmd, '-O', $dst_format;
6667         }
6668
6669         push @$cmd, $src_path;
6670
6671         if (!$dst_is_iscsi && $is_zero_initialized) {
6672             push @$cmd, "zeroinit:$dst_path";
6673         } else {
6674             push @$cmd, $dst_path;
6675         }
6676
6677         my $parser = sub {
6678             my $line = shift;
6679             if($line =~ m/\((\S+)\/100\%\)/){
6680                 my $percent = $1;
6681                 my $transferred = int($size * $percent / 100);
6682                 my $remaining = $size - $transferred;
6683
6684                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6685             }
6686
6687         };
6688
6689         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6690         my $err = $@;
6691         die "copy failed: $err" if $err;
6692     }
6693 }
6694
6695 sub qemu_img_format {
6696     my ($scfg, $volname) = @_;
6697
6698     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6699         return $1;
6700     } else {
6701         return "raw";
6702     }
6703 }
6704
6705 sub qemu_drive_mirror {
6706     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6707
6708     $jobs = {} if !$jobs;
6709
6710     my $qemu_target;
6711     my $format;
6712     $jobs->{"drive-$drive"} = {};
6713
6714     if ($dst_volid =~ /^nbd:/) {
6715         $qemu_target = $dst_volid;
6716         $format = "nbd";
6717     } else {
6718         my $storecfg = PVE::Storage::config();
6719         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6720
6721         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6722
6723         $format = qemu_img_format($dst_scfg, $dst_volname);
6724
6725         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6726
6727         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6728     }
6729
6730     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6731     $opts->{format} = $format if $format;
6732
6733     print "drive mirror is starting for drive-$drive\n";
6734
6735     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6736
6737     if (my $err = $@) {
6738         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6739         die "mirroring error: $err";
6740     }
6741
6742     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6743 }
6744
6745 sub qemu_drive_mirror_monitor {
6746     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6747
6748     eval {
6749         my $err_complete = 0;
6750
6751         while (1) {
6752             die "storage migration timed out\n" if $err_complete > 300;
6753
6754             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6755
6756             my $running_mirror_jobs = {};
6757             foreach my $stat (@$stats) {
6758                 next if $stat->{type} ne 'mirror';
6759                 $running_mirror_jobs->{$stat->{device}} = $stat;
6760             }
6761
6762             my $readycounter = 0;
6763
6764             foreach my $job (keys %$jobs) {
6765
6766                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6767                     print "$job : finished\n";
6768                     delete $jobs->{$job};
6769                     next;
6770                 }
6771
6772                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6773
6774                 my $busy = $running_mirror_jobs->{$job}->{busy};
6775                 my $ready = $running_mirror_jobs->{$job}->{ready};
6776                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6777                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6778                     my $remaining = $total - $transferred;
6779                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6780
6781                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6782                 }
6783
6784                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6785             }
6786
6787             last if scalar(keys %$jobs) == 0;
6788
6789             if ($readycounter == scalar(keys %$jobs)) {
6790                 print "all mirroring jobs are ready \n";
6791                 last if $skipcomplete; #do the complete later
6792
6793                 if ($vmiddst && $vmiddst != $vmid) {
6794                     my $agent_running = $qga && qga_check_running($vmid);
6795                     if ($agent_running) {
6796                         print "freeze filesystem\n";
6797                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6798                     } else {
6799                         print "suspend vm\n";
6800                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6801                     }
6802
6803                     # if we clone a disk for a new target vm, we don't switch the disk
6804                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6805
6806                     if ($agent_running) {
6807                         print "unfreeze filesystem\n";
6808                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6809                     } else {
6810                         print "resume vm\n";
6811                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6812                     }
6813
6814                     last;
6815                 } else {
6816
6817                     foreach my $job (keys %$jobs) {
6818                         # try to switch the disk if source and destination are on the same guest
6819                         print "$job: Completing block job...\n";
6820
6821                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6822                         if ($@ =~ m/cannot be completed/) {
6823                             print "$job: Block job cannot be completed, try again.\n";
6824                             $err_complete++;
6825                         }else {
6826                             print "$job: Completed successfully.\n";
6827                             $jobs->{$job}->{complete} = 1;
6828                         }
6829                     }
6830                 }
6831             }
6832             sleep 1;
6833         }
6834     };
6835     my $err = $@;
6836
6837     if ($err) {
6838         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6839         die "mirroring error: $err";
6840     }
6841
6842 }
6843
6844 sub qemu_blockjobs_cancel {
6845     my ($vmid, $jobs) = @_;
6846
6847     foreach my $job (keys %$jobs) {
6848         print "$job: Cancelling block job\n";
6849         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6850         $jobs->{$job}->{cancel} = 1;
6851     }
6852
6853     while (1) {
6854         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6855
6856         my $running_jobs = {};
6857         foreach my $stat (@$stats) {
6858             $running_jobs->{$stat->{device}} = $stat;
6859         }
6860
6861         foreach my $job (keys %$jobs) {
6862
6863             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6864                 print "$job: Done.\n";
6865                 delete $jobs->{$job};
6866             }
6867         }
6868
6869         last if scalar(keys %$jobs) == 0;
6870
6871         sleep 1;
6872     }
6873 }
6874
6875 sub clone_disk {
6876     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6877         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6878
6879     my $newvolid;
6880
6881     if (!$full) {
6882         print "create linked clone of drive $drivename ($drive->{file})\n";
6883         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6884         push @$newvollist, $newvolid;
6885     } else {
6886
6887         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6888         $storeid = $storage if $storage;
6889
6890         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6891         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6892
6893         print "create full clone of drive $drivename ($drive->{file})\n";
6894         my $name = undef;
6895         if (drive_is_cloudinit($drive)) {
6896             $name = "vm-$newvmid-cloudinit";
6897             $snapname = undef;
6898             # cloudinit only supports raw and qcow2 atm:
6899             if ($dst_format eq 'qcow2') {
6900                 $name .= '.qcow2';
6901             } elsif ($dst_format ne 'raw') {
6902                 die "clone: unhandled format for cloudinit image\n";
6903             }
6904         }
6905         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6906         push @$newvollist, $newvolid;
6907
6908         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6909
6910         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6911         if (!$running || $snapname) {
6912             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6913         } else {
6914
6915             my $kvmver = get_running_qemu_version ($vmid);
6916             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6917                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6918                     if $drive->{iothread};
6919             }
6920
6921             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6922         }
6923     }
6924
6925     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6926
6927     my $disk = $drive;
6928     $disk->{format} = undef;
6929     $disk->{file} = $newvolid;
6930     $disk->{size} = $size;
6931
6932     return $disk;
6933 }
6934
6935 # this only works if VM is running
6936 sub get_current_qemu_machine {
6937     my ($vmid) = @_;
6938
6939     my $cmd = { execute => 'query-machines', arguments => {} };
6940     my $res = vm_qmp_command($vmid, $cmd);
6941
6942     my ($current, $default);
6943     foreach my $e (@$res) {
6944         $default = $e->{name} if $e->{'is-default'};
6945         $current = $e->{name} if $e->{'is-current'};
6946     }
6947
6948     # fallback to the default machine if current is not supported by qemu
6949     return $current || $default || 'pc';
6950 }
6951
6952 sub get_running_qemu_version {
6953     my ($vmid) = @_;
6954     my $cmd = { execute => 'query-version', arguments => {} };
6955     my $res = vm_qmp_command($vmid, $cmd);
6956     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6957 }
6958
6959 sub qemu_machine_feature_enabled {
6960     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6961
6962     my $current_major;
6963     my $current_minor;
6964
6965     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6966
6967         $current_major = $3;
6968         $current_minor = $4;
6969
6970     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6971
6972         $current_major = $1;
6973         $current_minor = $2;
6974     }
6975
6976     return 1 if $current_major > $version_major ||
6977                 ($current_major == $version_major &&
6978                  $current_minor >= $version_minor);
6979 }
6980
6981 sub qemu_machine_pxe {
6982     my ($vmid, $conf, $machine) = @_;
6983
6984     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6985
6986     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6987         $machine .= '.pxe';
6988     }
6989
6990     return $machine;
6991 }
6992
6993 sub qemu_use_old_bios_files {
6994     my ($machine_type) = @_;
6995
6996     return if !$machine_type;
6997
6998     my $use_old_bios_files = undef;
6999
7000     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7001         $machine_type = $1;
7002         $use_old_bios_files = 1;
7003     } else {
7004         my $kvmver = kvm_user_version();
7005         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7006         # load new efi bios files on migration. So this hack is required to allow
7007         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7008         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7009         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7010     }
7011
7012     return ($use_old_bios_files, $machine_type);
7013 }
7014
7015 sub create_efidisk($$$$$) {
7016     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7017
7018     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7019     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7020
7021     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7022     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7023     PVE::Storage::activate_volumes($storecfg, [$volid]);
7024
7025     my $path = PVE::Storage::path($storecfg, $volid);
7026     eval {
7027         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7028     };
7029     die "Copying EFI vars image failed: $@" if $@;
7030
7031     return ($volid, $vars_size);
7032 }
7033
7034 sub vm_iothreads_list {
7035     my ($vmid) = @_;
7036
7037     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7038
7039     my $iothreads = {};
7040     foreach my $iothread (@$res) {
7041         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7042     }
7043
7044     return $iothreads;
7045 }
7046
7047 sub scsihw_infos {
7048     my ($conf, $drive) = @_;
7049
7050     my $maxdev = 0;
7051
7052     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7053         $maxdev = 7;
7054     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7055         $maxdev = 1;
7056     } else {
7057         $maxdev = 256;
7058     }
7059
7060     my $controller = int($drive->{index} / $maxdev);
7061     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7062
7063     return ($maxdev, $controller, $controller_prefix);
7064 }
7065
7066 sub add_hyperv_enlightenments {
7067     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7068
7069     return if $winversion < 6;
7070     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7071
7072     if ($gpu_passthrough || defined($hv_vendor_id)) {
7073         $hv_vendor_id //= 'proxmox';
7074         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7075     }
7076
7077     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7078         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7079         push @$cpuFlags , 'hv_vapic';
7080         push @$cpuFlags , 'hv_time';
7081     } else {
7082         push @$cpuFlags , 'hv_spinlocks=0xffff';
7083     }
7084
7085     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7086         push @$cpuFlags , 'hv_reset';
7087         push @$cpuFlags , 'hv_vpindex';
7088         push @$cpuFlags , 'hv_runtime';
7089     }
7090
7091     if ($winversion >= 7) {
7092         push @$cpuFlags , 'hv_relaxed';
7093
7094         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7095             push @$cpuFlags , 'hv_synic';
7096             push @$cpuFlags , 'hv_stimer';
7097         }
7098     }
7099 }
7100
7101 sub windows_version {
7102     my ($ostype) = @_;
7103
7104     return 0 if !$ostype;
7105
7106     my $winversion = 0;
7107
7108     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7109         $winversion = 5;
7110     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7111         $winversion = 6;
7112     } elsif ($ostype =~ m/^win(\d+)$/) {
7113         $winversion = $1;
7114     }
7115
7116     return $winversion;
7117 }
7118
7119 sub resolve_dst_disk_format {
7120         my ($storecfg, $storeid, $src_volname, $format) = @_;
7121         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7122
7123         if (!$format) {
7124             # if no target format is specified, use the source disk format as hint
7125             if ($src_volname) {
7126                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7127                 $format = qemu_img_format($scfg, $src_volname);
7128             } else {
7129                 return $defFormat;
7130             }
7131         }
7132
7133         # test if requested format is supported - else use default
7134         my $supported = grep { $_ eq $format } @$validFormats;
7135         $format = $defFormat if !$supported;
7136         return $format;
7137 }
7138
7139 sub resolve_first_disk {
7140     my $conf = shift;
7141     my @disks = PVE::QemuServer::valid_drive_names();
7142     my $firstdisk;
7143     foreach my $ds (reverse @disks) {
7144         next if !$conf->{$ds};
7145         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7146         next if PVE::QemuServer::drive_is_cdrom($disk);
7147         $firstdisk = $ds;
7148     }
7149     return $firstdisk;
7150 }
7151
7152 sub generate_uuid {
7153     my ($uuid, $uuid_str);
7154     UUID::generate($uuid);
7155     UUID::unparse($uuid, $uuid_str);
7156     return $uuid_str;
7157 }
7158
7159 sub generate_smbios1_uuid {
7160     return "uuid=".generate_uuid();
7161 }
7162
7163 sub nbd_stop {
7164     my ($vmid) = @_;
7165
7166     vm_mon_cmd($vmid, 'nbd-server-stop');
7167 }
7168
7169 # bash completion helper
7170
7171 sub complete_backup_archives {
7172     my ($cmdname, $pname, $cvalue) = @_;
7173
7174     my $cfg = PVE::Storage::config();
7175
7176     my $storeid;
7177
7178     if ($cvalue =~ m/^([^:]+):/) {
7179         $storeid = $1;
7180     }
7181
7182     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7183
7184     my $res = [];
7185     foreach my $id (keys %$data) {
7186         foreach my $item (@{$data->{$id}}) {
7187             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7188             push @$res, $item->{volid} if defined($item->{volid});
7189         }
7190     }
7191
7192     return $res;
7193 }
7194
7195 my $complete_vmid_full = sub {
7196     my ($running) = @_;
7197
7198     my $idlist = vmstatus();
7199
7200     my $res = [];
7201
7202     foreach my $id (keys %$idlist) {
7203         my $d = $idlist->{$id};
7204         if (defined($running)) {
7205             next if $d->{template};
7206             next if $running && $d->{status} ne 'running';
7207             next if !$running && $d->{status} eq 'running';
7208         }
7209         push @$res, $id;
7210
7211     }
7212     return $res;
7213 };
7214
7215 sub complete_vmid {
7216     return &$complete_vmid_full();
7217 }
7218
7219 sub complete_vmid_stopped {
7220     return &$complete_vmid_full(0);
7221 }
7222
7223 sub complete_vmid_running {
7224     return &$complete_vmid_full(1);
7225 }
7226
7227 sub complete_storage {
7228
7229     my $cfg = PVE::Storage::config();
7230     my $ids = $cfg->{ids};
7231
7232     my $res = [];
7233     foreach my $sid (keys %$ids) {
7234         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7235         next if !$ids->{$sid}->{content}->{images};
7236         push @$res, $sid;
7237     }
7238
7239     return $res;
7240 }
7241
7242 1;