PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::GuestHelpers;
  34 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  35 use PVE::QemuServer::Memory;
  36 use PVE::QemuServer::USB qw(parse_usb_device);
  37 use PVE::QemuServer::Cloudinit;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use Time::HiRes qw(gettimeofday);
  41 use File::Copy qw(copy);
  42 use URI::Escape;
  43
  44 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  45 my $OVMF = {
  46     x86_64 => [
  47         "$EDK2_FW_BASE/OVMF_CODE.fd",
  48         "$EDK2_FW_BASE/OVMF_VARS.fd"
  49     ],
  50     aarch64 => [
  51         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  52         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  53     ],
  54 };
  55
  56 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  57
  58 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  59
  60 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  61
  62 # Note about locking: we use flock on the config file protect
  63 # against concurent actions.
  64 # Aditionaly, we have a 'lock' setting in the config file. This
  65 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  66 # allowed when such lock is set. But you can ignore this kind of
  67 # lock with the --skiplock flag.
  68
  69 cfs_register_file('/qemu-server/',
  70                   \&parse_vm_config,
  71                   \&write_vm_config);
  72
  73 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  74     description => "Some command save/restore state from this location.",
  75     type => 'string',
  76     maxLength => 128,
  77     optional => 1,
  78 });
  79
  80 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  81     description => "The name of the snapshot.",
  82     type => 'string', format => 'pve-configid',
  83     maxLength => 40,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 mkdir "/etc/pve/nodes/$nodename";
 114 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 115 mkdir $confdir;
 116
 117 my $var_run_tmpdir = "/var/run/qemu-server";
 118 mkdir $var_run_tmpdir;
 119
 120 my $lock_dir = "/var/lock/qemu-server";
 121 mkdir $lock_dir;
 122
 123 my $cpu_vendor_list = {
 124     # Intel CPUs
 125     486 => 'GenuineIntel',
 126     pentium => 'GenuineIntel',
 127     pentium2  => 'GenuineIntel',
 128     pentium3  => 'GenuineIntel',
 129     coreduo => 'GenuineIntel',
 130     core2duo => 'GenuineIntel',
 131     Conroe  => 'GenuineIntel',
 132     Penryn  => 'GenuineIntel',
 133     Nehalem  => 'GenuineIntel',
 134     'Nehalem-IBRS'  => 'GenuineIntel',
 135     Westmere => 'GenuineIntel',
 136     'Westmere-IBRS' => 'GenuineIntel',
 137     SandyBridge => 'GenuineIntel',
 138     'SandyBridge-IBRS' => 'GenuineIntel',
 139     IvyBridge => 'GenuineIntel',
 140     'IvyBridge-IBRS' => 'GenuineIntel',
 141     Haswell => 'GenuineIntel',
 142     'Haswell-IBRS' => 'GenuineIntel',
 143     'Haswell-noTSX' => 'GenuineIntel',
 144     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 145     Broadwell => 'GenuineIntel',
 146     'Broadwell-IBRS' => 'GenuineIntel',
 147     'Broadwell-noTSX' => 'GenuineIntel',
 148     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 149     'Skylake-Client' => 'GenuineIntel',
 150     'Skylake-Client-IBRS' => 'GenuineIntel',
 151     'Skylake-Server' => 'GenuineIntel',
 152     'Skylake-Server-IBRS' => 'GenuineIntel',
 153
 154     # AMD CPUs
 155     athlon => 'AuthenticAMD',
 156     phenom  => 'AuthenticAMD',
 157     Opteron_G1  => 'AuthenticAMD',
 158     Opteron_G2  => 'AuthenticAMD',
 159     Opteron_G3  => 'AuthenticAMD',
 160     Opteron_G4  => 'AuthenticAMD',
 161     Opteron_G5  => 'AuthenticAMD',
 162     EPYC => 'AuthenticAMD',
 163     'EPYC-IBPB' => 'AuthenticAMD',
 164
 165     # generic types, use vendor from host node
 166     host => 'default',
 167     kvm32 => 'default',
 168     kvm64 => 'default',
 169     qemu32 => 'default',
 170     qemu64 => 'default',
 171     max => 'default',
 172 };
 173
 174 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 175
 176 my $cpu_fmt = {
 177     cputype => {
 178         description => "Emulated CPU type.",
 179         type => 'string',
 180         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 181         default => 'kvm64',
 182         default_key => 1,
 183     },
 184     hidden => {
 185         description => "Do not identify as a KVM virtual machine.",
 186         type => 'boolean',
 187         optional => 1,
 188         default => 0
 189     },
 190     'hv-vendor-id' => {
 191         type => 'string',
 192         pattern => qr/[a-zA-Z0-9]{1,12}/,
 193         format_description => 'vendor-id',
 194         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 195         optional => 1,
 196     },
 197     flags => {
 198         description => "List of additional CPU flags separated by ';'."
 199                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 200                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 201         format_description => '+FLAG[;-FLAG...]',
 202         type => 'string',
 203         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 204         optional => 1,
 205     },
 206 };
 207
 208 my $watchdog_fmt = {
 209     model => {
 210         default_key => 1,
 211         type => 'string',
 212         enum => [qw(i6300esb ib700)],
 213         description => "Watchdog type to emulate.",
 214         default => 'i6300esb',
 215         optional => 1,
 216     },
 217     action => {
 218         type => 'string',
 219         enum => [qw(reset shutdown poweroff pause debug none)],
 220         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 221         optional => 1,
 222     },
 223 };
 224 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 225
 226 my $agent_fmt = {
 227     enabled => {
 228         description => "Enable/disable Qemu GuestAgent.",
 229         type => 'boolean',
 230         default => 0,
 231         default_key => 1,
 232     },
 233     fstrim_cloned_disks => {
 234         description => "Run fstrim after cloning/moving a disk.",
 235         type => 'boolean',
 236         optional => 1,
 237         default => 0
 238     },
 239 };
 240
 241 my $vga_fmt = {
 242     type => {
 243         description => "Select the VGA type.",
 244         type => 'string',
 245         default => 'std',
 246         optional => 1,
 247         default_key => 1,
 248         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 249     },
 250     memory => {
 251         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 252         type => 'integer',
 253         optional => 1,
 254         minimum => 4,
 255         maximum => 512,
 256     },
 257 };
 258
 259 my $ivshmem_fmt = {
 260     size => {
 261         type => 'integer',
 262         minimum => 1,
 263         description => "The size of the file in MB.",
 264     },
 265     name => {
 266         type => 'string',
 267         pattern => '[a-zA-Z0-9\-]+',
 268         optional => 1,
 269         format_description => 'string',
 270         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6/3.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string',
 396         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 397         pattern => '[acdn]{1,4}',
 398         default => 'cdn',
 399     },
 400     bootdisk => {
 401         optional => 1,
 402         type => 'string', format => 'pve-qm-bootdisk',
 403         description => "Enable booting from specified disk.",
 404         pattern => '(ide|sata|scsi|virtio)\d+',
 405     },
 406     smp => {
 407         optional => 1,
 408         type => 'integer',
 409         description => "The number of CPUs. Please use option -sockets instead.",
 410         minimum => 1,
 411         default => 1,
 412     },
 413     sockets => {
 414         optional => 1,
 415         type => 'integer',
 416         description => "The number of CPU sockets.",
 417         minimum => 1,
 418         default => 1,
 419     },
 420     cores => {
 421         optional => 1,
 422         type => 'integer',
 423         description => "The number of cores per socket.",
 424         minimum => 1,
 425         default => 1,
 426     },
 427     numa => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Enable/disable NUMA.",
 431         default => 0,
 432     },
 433     hugepages => {
 434         optional => 1,
 435         type => 'string',
 436         description => "Enable/disable hugepages memory.",
 437         enum => [qw(any 2 1024)],
 438     },
 439     vcpus => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Number of hotplugged vcpus.",
 443         minimum => 1,
 444         default => 0,
 445     },
 446     acpi => {
 447         optional => 1,
 448         type => 'boolean',
 449         description => "Enable/disable ACPI.",
 450         default => 1,
 451     },
 452     agent => {
 453         optional => 1,
 454         description => "Enable/disable Qemu GuestAgent and its properties.",
 455         type => 'string',
 456         format => $agent_fmt,
 457     },
 458     kvm => {
 459         optional => 1,
 460         type => 'boolean',
 461         description => "Enable/disable KVM hardware virtualization.",
 462         default => 1,
 463     },
 464     tdf => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable time drift fix.",
 468         default => 0,
 469     },
 470     localtime => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 474     },
 475     freeze => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 479     },
 480     vga => {
 481         optional => 1,
 482         type => 'string', format => $vga_fmt,
 483         description => "Configure the VGA hardware.",
 484         verbose_description => "Configure the VGA Hardware. If you want to use ".
 485             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 486             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 487             "is 'std' for all OS types besides some Windows versions (XP and " .
 488             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 489             "display server. For win* OS you can select how many independent " .
 490             "displays you want, Linux guests can add displays them self.\n".
 491             "You can also run without any graphic card, using a serial device as terminal.",
 492     },
 493     watchdog => {
 494         optional => 1,
 495         type => 'string', format => 'pve-qm-watchdog',
 496         description => "Create a virtual hardware watchdog device.",
 497         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 498             " (by a guest action), the watchdog must be periodically polled " .
 499             "by an agent inside the guest or else the watchdog will reset " .
 500             "the guest (or execute the respective action specified)",
 501     },
 502     startdate => {
 503         optional => 1,
 504         type => 'string',
 505         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 506         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 507         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 508         default => 'now',
 509     },
 510     startup =>  get_standard_option('pve-startup-order'),
 511     template => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Enable/disable Template.",
 515         default => 0,
 516     },
 517     args => {
 518         optional => 1,
 519         type => 'string',
 520         description => "Arbitrary arguments passed to kvm.",
 521         verbose_description => <<EODESCR,
 522 Arbitrary arguments passed to kvm, for example:
 523
 524 args: -no-reboot -no-hpet
 525
 526 NOTE: this option is for experts only.
 527 EODESCR
 528     },
 529     tablet => {
 530         optional => 1,
 531         type => 'boolean',
 532         default => 1,
 533         description => "Enable/disable the USB tablet device.",
 534         verbose_description => "Enable/disable the USB tablet device. This device is " .
 535             "usually needed to allow absolute mouse positioning with VNC. " .
 536             "Else the mouse runs out of sync with normal VNC clients. " .
 537             "If you're running lots of console-only guests on one host, " .
 538             "you may consider disabling this to save some context switches. " .
 539             "This is turned off by default if you use spice (-vga=qxl).",
 540     },
 541     migrate_speed => {
 542         optional => 1,
 543         type => 'integer',
 544         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 545         minimum => 0,
 546         default => 0,
 547     },
 548     migrate_downtime => {
 549         optional => 1,
 550         type => 'number',
 551         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 552         minimum => 0,
 553         default => 0.1,
 554     },
 555     cdrom => {
 556         optional => 1,
 557         type => 'string', format => 'pve-qm-ide',
 558         typetext => '<volume>',
 559         description => "This is an alias for option -ide2",
 560     },
 561     cpu => {
 562         optional => 1,
 563         description => "Emulated CPU type.",
 564         type => 'string',
 565         format => $cpu_fmt,
 566     },
 567     parent => get_standard_option('pve-snapshot-name', {
 568         optional => 1,
 569         description => "Parent snapshot name. This is used internally, and should not be modified.",
 570     }),
 571     snaptime => {
 572         optional => 1,
 573         description => "Timestamp for snapshots.",
 574         type => 'integer',
 575         minimum => 0,
 576     },
 577     vmstate => {
 578         optional => 1,
 579         type => 'string', format => 'pve-volume-id',
 580         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 581     },
 582     vmstatestorage => get_standard_option('pve-storage-id', {
 583         description => "Default storage for VM state volumes/files.",
 584         optional => 1,
 585     }),
 586     runningmachine => get_standard_option('pve-qemu-machine', {
 587         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 588     }),
 589     machine => get_standard_option('pve-qemu-machine'),
 590     arch => {
 591         description => "Virtual processor architecture. Defaults to the host.",
 592         optional => 1,
 593         type => 'string',
 594         enum => [qw(x86_64 aarch64)],
 595     },
 596     smbios1 => {
 597         description => "Specify SMBIOS type 1 fields.",
 598         type => 'string', format => 'pve-qm-smbios1',
 599         maxLength => 256,
 600         optional => 1,
 601     },
 602     protection => {
 603         optional => 1,
 604         type => 'boolean',
 605         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 606         default => 0,
 607     },
 608     bios => {
 609         optional => 1,
 610         type => 'string',
 611         enum => [ qw(seabios ovmf) ],
 612         description => "Select BIOS implementation.",
 613         default => 'seabios',
 614     },
 615     vmgenid => {
 616         type => 'string',
 617         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 618         format_description => 'UUID',
 619         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 620         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 621             " 128-bit integer value identifier to the guest OS. This allows to".
 622             " notify the guest operating system when the virtual machine is".
 623             " executed with a different configuration (e.g. snapshot execution".
 624             " or creation from a template). The guest operating system notices".
 625             " the change, and is then able to react as appropriate by marking".
 626             " its copies of distributed databases as dirty, re-initializing its".
 627             " random number generator, etc.\n".
 628             "Note that auto-creation only works when done throug API/CLI create".
 629             " or update methods, but not when manually editing the config file.",
 630         default => "1 (autogenerated)",
 631         optional => 1,
 632     },
 633     hookscript => {
 634         type => 'string',
 635         format => 'pve-volume-id',
 636         optional => 1,
 637         description => "Script that will be executed during various steps in the vms lifetime.",
 638     },
 639     ivshmem => {
 640         type => 'string',
 641         format => $ivshmem_fmt,
 642         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 643         optional => 1,
 644     }
 645 };
 646
 647 my $cicustom_fmt = {
 648     meta => {
 649         type => 'string',
 650         optional => 1,
 651         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 652         format => 'pve-volume-id',
 653         format_description => 'volume',
 654     },
 655     network => {
 656         type => 'string',
 657         optional => 1,
 658         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 659         format => 'pve-volume-id',
 660         format_description => 'volume',
 661     },
 662     user => {
 663         type => 'string',
 664         optional => 1,
 665         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 666         format => 'pve-volume-id',
 667         format_description => 'volume',
 668     },
 669 };
 670 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 671
 672 my $confdesc_cloudinit = {
 673     citype => {
 674         optional => 1,
 675         type => 'string',
 676         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 677         enum => ['configdrive2', 'nocloud'],
 678     },
 679     ciuser => {
 680         optional => 1,
 681         type => 'string',
 682         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 683     },
 684     cipassword => {
 685         optional => 1,
 686         type => 'string',
 687         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 688     },
 689     cicustom => {
 690         optional => 1,
 691         type => 'string',
 692         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 693         format => 'pve-qm-cicustom',
 694     },
 695     searchdomain => {
 696         optional => 1,
 697         type => 'string',
 698         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 699     },
 700     nameserver => {
 701         optional => 1,
 702         type => 'string', format => 'address-list',
 703         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 704     },
 705     sshkeys => {
 706         optional => 1,
 707         type => 'string',
 708         format => 'urlencoded',
 709         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 710     },
 711 };
 712
 713 # what about other qemu settings ?
 714 #cpu => 'string',
 715 #machine => 'string',
 716 #fda => 'file',
 717 #fdb => 'file',
 718 #mtdblock => 'file',
 719 #sd => 'file',
 720 #pflash => 'file',
 721 #snapshot => 'bool',
 722 #bootp => 'file',
 723 ##tftp => 'dir',
 724 ##smb => 'dir',
 725 #kernel => 'file',
 726 #append => 'string',
 727 #initrd => 'file',
 728 ##soundhw => 'string',
 729
 730 while (my ($k, $v) = each %$confdesc) {
 731     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 732 }
 733
 734 my $MAX_IDE_DISKS = 4;
 735 my $MAX_SCSI_DISKS = 14;
 736 my $MAX_VIRTIO_DISKS = 16;
 737 my $MAX_SATA_DISKS = 6;
 738 my $MAX_USB_DEVICES = 5;
 739 my $MAX_NETS = 32;
 740 my $MAX_UNUSED_DISKS = 256;
 741 my $MAX_HOSTPCI_DEVICES = 4;
 742 my $MAX_SERIAL_PORTS = 4;
 743 my $MAX_PARALLEL_PORTS = 3;
 744 my $MAX_NUMA = 8;
 745
 746 my $numa_fmt = {
 747     cpus => {
 748         type => "string",
 749         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 750         description => "CPUs accessing this NUMA node.",
 751         format_description => "id[-id];...",
 752     },
 753     memory => {
 754         type => "number",
 755         description => "Amount of memory this NUMA node provides.",
 756         optional => 1,
 757     },
 758     hostnodes => {
 759         type => "string",
 760         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 761         description => "Host NUMA nodes to use.",
 762         format_description => "id[-id];...",
 763         optional => 1,
 764     },
 765     policy => {
 766         type => 'string',
 767         enum => [qw(preferred bind interleave)],
 768         description => "NUMA allocation policy.",
 769         optional => 1,
 770     },
 771 };
 772 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 773 my $numadesc = {
 774     optional => 1,
 775     type => 'string', format => $numa_fmt,
 776     description => "NUMA topology.",
 777 };
 778 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 779
 780 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 781     $confdesc->{"numa$i"} = $numadesc;
 782 }
 783
 784 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 785                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 786                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 787 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 788
 789 my $net_fmt_bridge_descr = <<__EOD__;
 790 Bridge to attach the network device to. The Proxmox VE standard bridge
 791 is called 'vmbr0'.
 792
 793 If you do not specify a bridge, we create a kvm user (NATed) network
 794 device, which provides DHCP and DNS services. The following addresses
 795 are used:
 796
 797  10.0.2.2   Gateway
 798  10.0.2.3   DNS Server
 799  10.0.2.4   SMB Server
 800
 801 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 802 __EOD__
 803
 804 my $net_fmt = {
 805     macaddr  => get_standard_option('mac-addr', {
 806         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 807     }),
 808     model => {
 809         type => 'string',
 810         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 811         enum => $nic_model_list,
 812         default_key => 1,
 813     },
 814     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 815     bridge => {
 816         type => 'string',
 817         description => $net_fmt_bridge_descr,
 818         format_description => 'bridge',
 819         optional => 1,
 820     },
 821     queues => {
 822         type => 'integer',
 823         minimum => 0, maximum => 16,
 824         description => 'Number of packet queues to be used on the device.',
 825         optional => 1,
 826     },
 827     rate => {
 828         type => 'number',
 829         minimum => 0,
 830         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 831         optional => 1,
 832     },
 833     tag => {
 834         type => 'integer',
 835         minimum => 1, maximum => 4094,
 836         description => 'VLAN tag to apply to packets on this interface.',
 837         optional => 1,
 838     },
 839     trunks => {
 840         type => 'string',
 841         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 842         description => 'VLAN trunks to pass through this interface.',
 843         format_description => 'vlanid[;vlanid...]',
 844         optional => 1,
 845     },
 846     firewall => {
 847         type => 'boolean',
 848         description => 'Whether this interface should be protected by the firewall.',
 849         optional => 1,
 850     },
 851     link_down => {
 852         type => 'boolean',
 853         description => 'Whether this interface should be disconnected (like pulling the plug).',
 854         optional => 1,
 855     },
 856 };
 857
 858 my $netdesc = {
 859     optional => 1,
 860     type => 'string', format => $net_fmt,
 861     description => "Specify network devices.",
 862 };
 863
 864 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 865
 866 my $ipconfig_fmt = {
 867     ip => {
 868         type => 'string',
 869         format => 'pve-ipv4-config',
 870         format_description => 'IPv4Format/CIDR',
 871         description => 'IPv4 address in CIDR format.',
 872         optional => 1,
 873         default => 'dhcp',
 874     },
 875     gw => {
 876         type => 'string',
 877         format => 'ipv4',
 878         format_description => 'GatewayIPv4',
 879         description => 'Default gateway for IPv4 traffic.',
 880         optional => 1,
 881         requires => 'ip',
 882     },
 883     ip6 => {
 884         type => 'string',
 885         format => 'pve-ipv6-config',
 886         format_description => 'IPv6Format/CIDR',
 887         description => 'IPv6 address in CIDR format.',
 888         optional => 1,
 889         default => 'dhcp',
 890     },
 891     gw6 => {
 892         type => 'string',
 893         format => 'ipv6',
 894         format_description => 'GatewayIPv6',
 895         description => 'Default gateway for IPv6 traffic.',
 896         optional => 1,
 897         requires => 'ip6',
 898     },
 899 };
 900 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 901 my $ipconfigdesc = {
 902     optional => 1,
 903     type => 'string', format => 'pve-qm-ipconfig',
 904     description => <<'EODESCR',
 905 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 906
 907 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 908
 909 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 910 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 911
 912 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 913 EODESCR
 914 };
 915 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 916
 917 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 918     $confdesc->{"net$i"} = $netdesc;
 919     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 920 }
 921
 922 foreach my $key (keys %$confdesc_cloudinit) {
 923     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 924 }
 925
 926 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 927 sub verify_volume_id_or_qm_path {
 928     my ($volid, $noerr) = @_;
 929
 930     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 931         return $volid;
 932     }
 933
 934     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 935     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 936     if ($@) {
 937         return undef if $noerr;
 938         die $@;
 939     }
 940     return $volid;
 941 }
 942
 943 my $drivename_hash;
 944
 945 my %drivedesc_base = (
 946     volume => { alias => 'file' },
 947     file => {
 948         type => 'string',
 949         format => 'pve-volume-id-or-qm-path',
 950         default_key => 1,
 951         format_description => 'volume',
 952         description => "The drive's backing volume.",
 953     },
 954     media => {
 955         type => 'string',
 956         enum => [qw(cdrom disk)],
 957         description => "The drive's media type.",
 958         default => 'disk',
 959         optional => 1
 960     },
 961     cyls => {
 962         type => 'integer',
 963         description => "Force the drive's physical geometry to have a specific cylinder count.",
 964         optional => 1
 965     },
 966     heads => {
 967         type => 'integer',
 968         description => "Force the drive's physical geometry to have a specific head count.",
 969         optional => 1
 970     },
 971     secs => {
 972         type => 'integer',
 973         description => "Force the drive's physical geometry to have a specific sector count.",
 974         optional => 1
 975     },
 976     trans => {
 977         type => 'string',
 978         enum => [qw(none lba auto)],
 979         description => "Force disk geometry bios translation mode.",
 980         optional => 1,
 981     },
 982     snapshot => {
 983         type => 'boolean',
 984         description => "Controls qemu's snapshot mode feature."
 985             . " If activated, changes made to the disk are temporary and will"
 986             . " be discarded when the VM is shutdown.",
 987         optional => 1,
 988     },
 989     cache => {
 990         type => 'string',
 991         enum => [qw(none writethrough writeback unsafe directsync)],
 992         description => "The drive's cache mode",
 993         optional => 1,
 994     },
 995     format => get_standard_option('pve-qm-image-format'),
 996     size => {
 997         type => 'string',
 998         format => 'disk-size',
 999         format_description => 'DiskSize',
1000         description => "Disk size. This is purely informational and has no effect.",
1001         optional => 1,
1002     },
1003     backup => {
1004         type => 'boolean',
1005         description => "Whether the drive should be included when making backups.",
1006         optional => 1,
1007     },
1008     replicate => {
1009         type => 'boolean',
1010         description => 'Whether the drive should considered for replication jobs.',
1011         optional => 1,
1012         default => 1,
1013     },
1014     rerror => {
1015         type => 'string',
1016         enum => [qw(ignore report stop)],
1017         description => 'Read error action.',
1018         optional => 1,
1019     },
1020     werror => {
1021         type => 'string',
1022         enum => [qw(enospc ignore report stop)],
1023         description => 'Write error action.',
1024         optional => 1,
1025     },
1026     aio => {
1027         type => 'string',
1028         enum => [qw(native threads)],
1029         description => 'AIO type to use.',
1030         optional => 1,
1031     },
1032     discard => {
1033         type => 'string',
1034         enum => [qw(ignore on)],
1035         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1036         optional => 1,
1037     },
1038     detect_zeroes => {
1039         type => 'boolean',
1040         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1041         optional => 1,
1042     },
1043     serial => {
1044         type => 'string',
1045         format => 'urlencoded',
1046         format_description => 'serial',
1047         maxLength => 20*3, # *3 since it's %xx url enoded
1048         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1049         optional => 1,
1050     },
1051     shared => {
1052         type => 'boolean',
1053         description => 'Mark this locally-managed volume as available on all nodes',
1054         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1055         optional => 1,
1056         default => 0,
1057     }
1058 );
1059
1060 my %iothread_fmt = ( iothread => {
1061         type => 'boolean',
1062         description => "Whether to use iothreads for this drive",
1063         optional => 1,
1064 });
1065
1066 my %model_fmt = (
1067     model => {
1068         type => 'string',
1069         format => 'urlencoded',
1070         format_description => 'model',
1071         maxLength => 40*3, # *3 since it's %xx url enoded
1072         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1073         optional => 1,
1074     },
1075 );
1076
1077 my %queues_fmt = (
1078     queues => {
1079         type => 'integer',
1080         description => "Number of queues.",
1081         minimum => 2,
1082         optional => 1
1083     }
1084 );
1085
1086 my %scsiblock_fmt = (
1087     scsiblock => {
1088         type => 'boolean',
1089         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1090         optional => 1,
1091         default => 0,
1092     },
1093 );
1094
1095 my %ssd_fmt = (
1096     ssd => {
1097         type => 'boolean',
1098         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1099         optional => 1,
1100     },
1101 );
1102
1103 my %wwn_fmt = (
1104     wwn => {
1105         type => 'string',
1106         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1107         format_description => 'wwn',
1108         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1109         optional => 1,
1110     },
1111 );
1112
1113 my $add_throttle_desc = sub {
1114     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1115     my $d = {
1116         type => $type,
1117         format_description => $unit,
1118         description => "Maximum $what in $longunit.",
1119         optional => 1,
1120     };
1121     $d->{minimum} = $minimum if defined($minimum);
1122     $drivedesc_base{$key} = $d;
1123 };
1124 # throughput: (leaky bucket)
1125 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1126 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1127 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1128 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1129 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1130 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1131 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1132 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1133 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1134
1135 # pools: (pool of IO before throttling starts taking effect)
1136 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1137 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1138 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1139 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1140 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1141 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1142
1143 # burst lengths
1144 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1145 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1146 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1147 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1148 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1149 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1150
1151 # legacy support
1152 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1153 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1154 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1155 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1156
1157 my $ide_fmt = {
1158     %drivedesc_base,
1159     %model_fmt,
1160     %ssd_fmt,
1161     %wwn_fmt,
1162 };
1163 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1164
1165 my $idedesc = {
1166     optional => 1,
1167     type => 'string', format => $ide_fmt,
1168     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1169 };
1170 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1171
1172 my $scsi_fmt = {
1173     %drivedesc_base,
1174     %iothread_fmt,
1175     %queues_fmt,
1176     %scsiblock_fmt,
1177     %ssd_fmt,
1178     %wwn_fmt,
1179 };
1180 my $scsidesc = {
1181     optional => 1,
1182     type => 'string', format => $scsi_fmt,
1183     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1184 };
1185 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1186
1187 my $sata_fmt = {
1188     %drivedesc_base,
1189     %ssd_fmt,
1190     %wwn_fmt,
1191 };
1192 my $satadesc = {
1193     optional => 1,
1194     type => 'string', format => $sata_fmt,
1195     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1196 };
1197 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1198
1199 my $virtio_fmt = {
1200     %drivedesc_base,
1201     %iothread_fmt,
1202 };
1203 my $virtiodesc = {
1204     optional => 1,
1205     type => 'string', format => $virtio_fmt,
1206     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1207 };
1208 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1209
1210 my $alldrive_fmt = {
1211     %drivedesc_base,
1212     %iothread_fmt,
1213     %model_fmt,
1214     %queues_fmt,
1215     %scsiblock_fmt,
1216     %ssd_fmt,
1217     %wwn_fmt,
1218 };
1219
1220 my $efidisk_fmt = {
1221     volume => { alias => 'file' },
1222     file => {
1223         type => 'string',
1224         format => 'pve-volume-id-or-qm-path',
1225         default_key => 1,
1226         format_description => 'volume',
1227         description => "The drive's backing volume.",
1228     },
1229     format => get_standard_option('pve-qm-image-format'),
1230     size => {
1231         type => 'string',
1232         format => 'disk-size',
1233         format_description => 'DiskSize',
1234         description => "Disk size. This is purely informational and has no effect.",
1235         optional => 1,
1236     },
1237 };
1238
1239 my $efidisk_desc = {
1240     optional => 1,
1241     type => 'string', format => $efidisk_fmt,
1242     description => "Configure a Disk for storing EFI vars",
1243 };
1244
1245 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1246
1247 my $usb_fmt = {
1248     host => {
1249         default_key => 1,
1250         type => 'string', format => 'pve-qm-usb-device',
1251         format_description => 'HOSTUSBDEVICE|spice',
1252         description => <<EODESCR,
1253 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1254
1255  'bus-port(.port)*' (decimal numbers) or
1256  'vendor_id:product_id' (hexadeciaml numbers) or
1257  'spice'
1258
1259 You can use the 'lsusb -t' command to list existing usb devices.
1260
1261 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1262
1263 The value 'spice' can be used to add a usb redirection devices for spice.
1264 EODESCR
1265     },
1266     usb3 => {
1267         optional => 1,
1268         type => 'boolean',
1269         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1270         default => 0,
1271     },
1272 };
1273
1274 my $usbdesc = {
1275     optional => 1,
1276     type => 'string', format => $usb_fmt,
1277     description => "Configure an USB device (n is 0 to 4).",
1278 };
1279 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1280
1281 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1282 my $hostpci_fmt = {
1283     host => {
1284         default_key => 1,
1285         type => 'string',
1286         pattern => qr/$PCIRE(;$PCIRE)*/,
1287         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1288         description => <<EODESCR,
1289 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1290 of PCI virtual functions of the host. HOSTPCIID syntax is:
1291
1292 'bus:dev.func' (hexadecimal numbers)
1293
1294 You can us the 'lspci' command to list existing PCI devices.
1295 EODESCR
1296     },
1297     rombar => {
1298         type => 'boolean',
1299         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1300         optional => 1,
1301         default => 1,
1302     },
1303     romfile => {
1304         type => 'string',
1305         pattern => '[^,;]+',
1306         format_description => 'string',
1307         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1308         optional => 1,
1309     },
1310     pcie => {
1311         type => 'boolean',
1312         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1313         optional => 1,
1314         default => 0,
1315     },
1316     'x-vga' => {
1317         type => 'boolean',
1318         description =>  "Enable vfio-vga device support.",
1319         optional => 1,
1320         default => 0,
1321     },
1322     'mdev' => {
1323         type => 'string',
1324         format_description => 'string',
1325         pattern => '[^/\.:]+',
1326         optional => 1,
1327         description => <<EODESCR
1328 The type of mediated device to use.
1329 An instance of this type will be created on startup of the VM and
1330 will be cleaned up when the VM stops.
1331 EODESCR
1332     }
1333 };
1334 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1335
1336 my $hostpcidesc = {
1337         optional => 1,
1338         type => 'string', format => 'pve-qm-hostpci',
1339         description => "Map host PCI devices into guest.",
1340         verbose_description =>  <<EODESCR,
1341 Map host PCI devices into guest.
1342
1343 NOTE: This option allows direct access to host hardware. So it is no longer
1344 possible to migrate such machines - use with special care.
1345
1346 CAUTION: Experimental! User reported problems with this option.
1347 EODESCR
1348 };
1349 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1350
1351 my $serialdesc = {
1352         optional => 1,
1353         type => 'string',
1354         pattern => '(/dev/.+|socket)',
1355         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1356         verbose_description =>  <<EODESCR,
1357 Create a serial device inside the VM (n is 0 to 3), and pass through a
1358 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1359 host side (use 'qm terminal' to open a terminal connection).
1360
1361 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1362
1363 CAUTION: Experimental! User reported problems with this option.
1364 EODESCR
1365 };
1366
1367 my $paralleldesc= {
1368         optional => 1,
1369         type => 'string',
1370         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1371         description =>  "Map host parallel devices (n is 0 to 2).",
1372         verbose_description =>  <<EODESCR,
1373 Map host parallel devices (n is 0 to 2).
1374
1375 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1376
1377 CAUTION: Experimental! User reported problems with this option.
1378 EODESCR
1379 };
1380
1381 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1382     $confdesc->{"parallel$i"} = $paralleldesc;
1383 }
1384
1385 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1386     $confdesc->{"serial$i"} = $serialdesc;
1387 }
1388
1389 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1390     $confdesc->{"hostpci$i"} = $hostpcidesc;
1391 }
1392
1393 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1394     $drivename_hash->{"ide$i"} = 1;
1395     $confdesc->{"ide$i"} = $idedesc;
1396 }
1397
1398 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1399     $drivename_hash->{"sata$i"} = 1;
1400     $confdesc->{"sata$i"} = $satadesc;
1401 }
1402
1403 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1404     $drivename_hash->{"scsi$i"} = 1;
1405     $confdesc->{"scsi$i"} = $scsidesc ;
1406 }
1407
1408 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1409     $drivename_hash->{"virtio$i"} = 1;
1410     $confdesc->{"virtio$i"} = $virtiodesc;
1411 }
1412
1413 $drivename_hash->{efidisk0} = 1;
1414 $confdesc->{efidisk0} = $efidisk_desc;
1415
1416 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1417     $confdesc->{"usb$i"} = $usbdesc;
1418 }
1419
1420 my $unuseddesc = {
1421     optional => 1,
1422     type => 'string', format => 'pve-volume-id',
1423     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1424 };
1425
1426 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1427     $confdesc->{"unused$i"} = $unuseddesc;
1428 }
1429
1430 my $kvm_api_version = 0;
1431
1432 sub kvm_version {
1433     return $kvm_api_version if $kvm_api_version;
1434
1435     open my $fh, '<', '/dev/kvm'
1436         or return undef;
1437
1438     # 0xae00 => KVM_GET_API_VERSION
1439     $kvm_api_version = ioctl($fh, 0xae00, 0);
1440
1441     return $kvm_api_version;
1442 }
1443
1444 my $kvm_user_version;
1445
1446 sub kvm_user_version {
1447
1448     return $kvm_user_version if $kvm_user_version;
1449
1450     $kvm_user_version = 'unknown';
1451
1452     my $code = sub {
1453         my $line = shift;
1454         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1455             $kvm_user_version = $2;
1456         }
1457     };
1458
1459     eval { run_command("kvm -version", outfunc => $code); };
1460     warn $@ if $@;
1461
1462     return $kvm_user_version;
1463
1464 }
1465
1466 sub kernel_has_vhost_net {
1467     return -c '/dev/vhost-net';
1468 }
1469
1470 sub valid_drive_names {
1471     # order is important - used to autoselect boot disk
1472     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1473             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1474             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1475             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1476             'efidisk0');
1477 }
1478
1479 sub is_valid_drivename {
1480     my $dev = shift;
1481
1482     return defined($drivename_hash->{$dev});
1483 }
1484
1485 sub option_exists {
1486     my $key = shift;
1487     return defined($confdesc->{$key});
1488 }
1489
1490 sub nic_models {
1491     return $nic_model_list;
1492 }
1493
1494 sub os_list_description {
1495
1496     return {
1497         other => 'Other',
1498         wxp => 'Windows XP',
1499         w2k => 'Windows 2000',
1500         w2k3 =>, 'Windows 2003',
1501         w2k8 => 'Windows 2008',
1502         wvista => 'Windows Vista',
1503         win7 => 'Windows 7',
1504         win8 => 'Windows 8/2012',
1505         win10 => 'Windows 10/2016',
1506         l24 => 'Linux 2.4',
1507         l26 => 'Linux 2.6',
1508     };
1509 }
1510
1511 my $cdrom_path;
1512
1513 sub get_cdrom_path {
1514
1515     return  $cdrom_path if $cdrom_path;
1516
1517     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1518     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1519     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1520 }
1521
1522 sub get_iso_path {
1523     my ($storecfg, $vmid, $cdrom) = @_;
1524
1525     if ($cdrom eq 'cdrom') {
1526         return get_cdrom_path();
1527     } elsif ($cdrom eq 'none') {
1528         return '';
1529     } elsif ($cdrom =~ m|^/|) {
1530         return $cdrom;
1531     } else {
1532         return PVE::Storage::path($storecfg, $cdrom);
1533     }
1534 }
1535
1536 # try to convert old style file names to volume IDs
1537 sub filename_to_volume_id {
1538     my ($vmid, $file, $media) = @_;
1539
1540      if (!($file eq 'none' || $file eq 'cdrom' ||
1541           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1542
1543         return undef if $file =~ m|/|;
1544
1545         if ($media && $media eq 'cdrom') {
1546             $file = "local:iso/$file";
1547         } else {
1548             $file = "local:$vmid/$file";
1549         }
1550     }
1551
1552     return $file;
1553 }
1554
1555 sub verify_media_type {
1556     my ($opt, $vtype, $media) = @_;
1557
1558     return if !$media;
1559
1560     my $etype;
1561     if ($media eq 'disk') {
1562         $etype = 'images';
1563     } elsif ($media eq 'cdrom') {
1564         $etype = 'iso';
1565     } else {
1566         die "internal error";
1567     }
1568
1569     return if ($vtype eq $etype);
1570
1571     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1572 }
1573
1574 sub cleanup_drive_path {
1575     my ($opt, $storecfg, $drive) = @_;
1576
1577     # try to convert filesystem paths to volume IDs
1578
1579     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1580         ($drive->{file} !~ m|^/dev/.+|) &&
1581         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1582         ($drive->{file} !~ m/^\d+$/)) {
1583         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1584         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1585         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1586         verify_media_type($opt, $vtype, $drive->{media});
1587         $drive->{file} = $volid;
1588     }
1589
1590     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1591 }
1592
1593 sub parse_hotplug_features {
1594     my ($data) = @_;
1595
1596     my $res = {};
1597
1598     return $res if $data eq '0';
1599
1600     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1601
1602     foreach my $feature (PVE::Tools::split_list($data)) {
1603         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1604             $res->{$1} = 1;
1605         } else {
1606             die "invalid hotplug feature '$feature'\n";
1607         }
1608     }
1609     return $res;
1610 }
1611
1612 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1613 sub pve_verify_hotplug_features {
1614     my ($value, $noerr) = @_;
1615
1616     return $value if parse_hotplug_features($value);
1617
1618     return undef if $noerr;
1619
1620     die "unable to parse hotplug option\n";
1621 }
1622
1623 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1624 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1625 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1626 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1627 #        [,iothread=on][,serial=serial][,model=model]
1628
1629 sub parse_drive {
1630     my ($key, $data) = @_;
1631
1632     my ($interface, $index);
1633
1634     if ($key =~ m/^([^\d]+)(\d+)$/) {
1635         $interface = $1;
1636         $index = $2;
1637     } else {
1638         return undef;
1639     }
1640
1641     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1642                                      : $confdesc->{$key}->{format};
1643     if (!$desc) {
1644         warn "invalid drive key: $key\n";
1645         return undef;
1646     }
1647     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1648     return undef if !$res;
1649     $res->{interface} = $interface;
1650     $res->{index} = $index;
1651
1652     my $error = 0;
1653     foreach my $opt (qw(bps bps_rd bps_wr)) {
1654         if (my $bps = defined(delete $res->{$opt})) {
1655             if (defined($res->{"m$opt"})) {
1656                 warn "both $opt and m$opt specified\n";
1657                 ++$error;
1658                 next;
1659             }
1660             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1661         }
1662     }
1663
1664     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1665     for my $requirement (
1666         [mbps_max => 'mbps'],
1667         [mbps_rd_max => 'mbps_rd'],
1668         [mbps_wr_max => 'mbps_wr'],
1669         [miops_max => 'miops'],
1670         [miops_rd_max => 'miops_rd'],
1671         [miops_wr_max => 'miops_wr'],
1672         [bps_max_length => 'mbps_max'],
1673         [bps_rd_max_length => 'mbps_rd_max'],
1674         [bps_wr_max_length => 'mbps_wr_max'],
1675         [iops_max_length => 'iops_max'],
1676         [iops_rd_max_length => 'iops_rd_max'],
1677         [iops_wr_max_length => 'iops_wr_max']) {
1678         my ($option, $requires) = @$requirement;
1679         if ($res->{$option} && !$res->{$requires}) {
1680             warn "$option requires $requires\n";
1681             ++$error;
1682         }
1683     }
1684
1685     return undef if $error;
1686
1687     return undef if $res->{mbps_rd} && $res->{mbps};
1688     return undef if $res->{mbps_wr} && $res->{mbps};
1689     return undef if $res->{iops_rd} && $res->{iops};
1690     return undef if $res->{iops_wr} && $res->{iops};
1691
1692     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1693         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1694         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1695         return undef if $res->{interface} eq 'virtio';
1696     }
1697
1698     if (my $size = $res->{size}) {
1699         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1700     }
1701
1702     return $res;
1703 }
1704
1705 sub print_drive {
1706     my ($vmid, $drive) = @_;
1707     my $data = { %$drive };
1708     delete $data->{$_} for qw(index interface);
1709     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1710 }
1711
1712 sub scsi_inquiry {
1713     my($fh, $noerr) = @_;
1714
1715     my $SG_IO = 0x2285;
1716     my $SG_GET_VERSION_NUM = 0x2282;
1717
1718     my $versionbuf = "\x00" x 8;
1719     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1720     if (!$ret) {
1721         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1722         return undef;
1723     }
1724     my $version = unpack("I", $versionbuf);
1725     if ($version < 30000) {
1726         die "scsi generic interface too old\n"  if !$noerr;
1727         return undef;
1728     }
1729
1730     my $buf = "\x00" x 36;
1731     my $sensebuf = "\x00" x 8;
1732     my $cmd = pack("C x3 C x1", 0x12, 36);
1733
1734     # see /usr/include/scsi/sg.h
1735     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1736
1737     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1738                       length($sensebuf), 0, length($buf), $buf,
1739                       $cmd, $sensebuf, 6000);
1740
1741     $ret = ioctl($fh, $SG_IO, $packet);
1742     if (!$ret) {
1743         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1744         return undef;
1745     }
1746
1747     my @res = unpack($sg_io_hdr_t, $packet);
1748     if ($res[17] || $res[18]) {
1749         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1750         return undef;
1751     }
1752
1753     my $res = {};
1754     (my $byte0, my $byte1, $res->{vendor},
1755      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1756
1757     $res->{removable} = $byte1 & 128 ? 1 : 0;
1758     $res->{type} = $byte0 & 31;
1759
1760     return $res;
1761 }
1762
1763 sub path_is_scsi {
1764     my ($path) = @_;
1765
1766     my $fh = IO::File->new("+<$path") || return undef;
1767     my $res = scsi_inquiry($fh, 1);
1768     close($fh);
1769
1770     return $res;
1771 }
1772
1773 sub machine_type_is_q35 {
1774     my ($conf) = @_;
1775
1776     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1777 }
1778
1779 sub print_tabletdevice_full {
1780     my ($conf, $arch) = @_;
1781
1782     my $q35 = machine_type_is_q35($conf);
1783
1784     # we use uhci for old VMs because tablet driver was buggy in older qemu
1785     my $usbbus;
1786     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1787         $usbbus = 'ehci';
1788     } else {
1789         $usbbus = 'uhci';
1790     }
1791
1792     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1793 }
1794
1795 sub print_keyboarddevice_full {
1796     my ($conf, $arch, $machine) = @_;
1797
1798     return undef if $arch ne 'aarch64';
1799
1800     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1801 }
1802
1803 sub print_drivedevice_full {
1804     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1805
1806     my $device = '';
1807     my $maxdev = 0;
1808
1809     if ($drive->{interface} eq 'virtio') {
1810         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1811         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1812         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1813     } elsif ($drive->{interface} eq 'scsi') {
1814
1815         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1816         my $unit = $drive->{index} % $maxdev;
1817         my $devicetype = 'hd';
1818         my $path = '';
1819         if (drive_is_cdrom($drive)) {
1820             $devicetype = 'cd';
1821         } else {
1822             if ($drive->{file} =~ m|^/|) {
1823                 $path = $drive->{file};
1824                 if (my $info = path_is_scsi($path)) {
1825                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1826                         $devicetype = 'block';
1827                     } elsif ($info->{type} == 1) { # tape
1828                         $devicetype = 'generic';
1829                     }
1830                 }
1831             } else {
1832                  $path = PVE::Storage::path($storecfg, $drive->{file});
1833             }
1834
1835             if($path =~ m/^iscsi\:\/\//){
1836                 $devicetype = 'generic';
1837             }
1838         }
1839
1840         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1841             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1842         } else {
1843             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1844         }
1845
1846         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1847             $device .= ",rotation_rate=1";
1848         }
1849         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1850
1851     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1852         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1853         my $controller = int($drive->{index} / $maxdev);
1854         my $unit = $drive->{index} % $maxdev;
1855         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1856
1857         $device = "ide-$devicetype";
1858         if ($drive->{interface} eq 'ide') {
1859             $device .= ",bus=ide.$controller,unit=$unit";
1860         } else {
1861             $device .= ",bus=ahci$controller.$unit";
1862         }
1863         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1864
1865         if ($devicetype eq 'hd') {
1866             if (my $model = $drive->{model}) {
1867                 $model = URI::Escape::uri_unescape($model);
1868                 $device .= ",model=$model";
1869             }
1870             if ($drive->{ssd}) {
1871                 $device .= ",rotation_rate=1";
1872             }
1873         }
1874         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1875     } elsif ($drive->{interface} eq 'usb') {
1876         die "implement me";
1877         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1878     } else {
1879         die "unsupported interface type";
1880     }
1881
1882     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1883
1884     if (my $serial = $drive->{serial}) {
1885         $serial = URI::Escape::uri_unescape($serial);
1886         $device .= ",serial=$serial";
1887     }
1888
1889
1890     return $device;
1891 }
1892
1893 sub get_initiator_name {
1894     my $initiator;
1895
1896     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1897     while (defined(my $line = <$fh>)) {
1898         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1899         $initiator = $1;
1900         last;
1901     }
1902     $fh->close();
1903
1904     return $initiator;
1905 }
1906
1907 sub print_drive_full {
1908     my ($storecfg, $vmid, $drive) = @_;
1909
1910     my $path;
1911     my $volid = $drive->{file};
1912     my $format;
1913
1914     if (drive_is_cdrom($drive)) {
1915         $path = get_iso_path($storecfg, $vmid, $volid);
1916     } else {
1917         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1918         if ($storeid) {
1919             $path = PVE::Storage::path($storecfg, $volid);
1920             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1921             $format = qemu_img_format($scfg, $volname);
1922         } else {
1923             $path = $volid;
1924             $format = "raw";
1925         }
1926    }
1927
1928     my $opts = '';
1929     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1930     foreach my $o (@qemu_drive_options) {
1931         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1932     }
1933
1934     # snapshot only accepts on|off
1935     if (defined($drive->{snapshot})) {
1936         my $v = $drive->{snapshot} ? 'on' : 'off';
1937         $opts .= ",snapshot=$v";
1938     }
1939
1940     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1941         my ($dir, $qmpname) = @$type;
1942         if (my $v = $drive->{"mbps$dir"}) {
1943             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1944         }
1945         if (my $v = $drive->{"mbps${dir}_max"}) {
1946             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1947         }
1948         if (my $v = $drive->{"bps${dir}_max_length"}) {
1949             $opts .= ",throttling.bps$qmpname-max-length=$v";
1950         }
1951         if (my $v = $drive->{"iops${dir}"}) {
1952             $opts .= ",throttling.iops$qmpname=$v";
1953         }
1954         if (my $v = $drive->{"iops${dir}_max"}) {
1955             $opts .= ",throttling.iops$qmpname-max=$v";
1956         }
1957         if (my $v = $drive->{"iops${dir}_max_length"}) {
1958             $opts .= ",throttling.iops$qmpname-max-length=$v";
1959         }
1960     }
1961
1962     $opts .= ",format=$format" if $format && !$drive->{format};
1963
1964     my $cache_direct = 0;
1965
1966     if (my $cache = $drive->{cache}) {
1967         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1968     } elsif (!drive_is_cdrom($drive)) {
1969         $opts .= ",cache=none";
1970         $cache_direct = 1;
1971     }
1972
1973     # aio native works only with O_DIRECT
1974     if (!$drive->{aio}) {
1975         if($cache_direct) {
1976             $opts .= ",aio=native";
1977         } else {
1978             $opts .= ",aio=threads";
1979         }
1980     }
1981
1982     if (!drive_is_cdrom($drive)) {
1983         my $detectzeroes;
1984         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1985             $detectzeroes = 'off';
1986         } elsif ($drive->{discard}) {
1987             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1988         } else {
1989             # This used to be our default with discard not being specified:
1990             $detectzeroes = 'on';
1991         }
1992         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1993     }
1994
1995     my $pathinfo = $path ? "file=$path," : '';
1996
1997     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1998 }
1999
2000 sub print_netdevice_full {
2001     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2002
2003     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2004
2005     my $device = $net->{model};
2006     if ($net->{model} eq 'virtio') {
2007          $device = 'virtio-net-pci';
2008      };
2009
2010     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2011     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2012     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2013         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2014         my $vectors = $net->{queues} * 2 + 2;
2015         $tmpstr .= ",vectors=$vectors,mq=on";
2016     }
2017     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2018
2019     if ($use_old_bios_files) {
2020         my $romfile;
2021         if ($device eq 'virtio-net-pci') {
2022             $romfile = 'pxe-virtio.rom';
2023         } elsif ($device eq 'e1000') {
2024             $romfile = 'pxe-e1000.rom';
2025         } elsif ($device eq 'ne2k') {
2026             $romfile = 'pxe-ne2k_pci.rom';
2027         } elsif ($device eq 'pcnet') {
2028             $romfile = 'pxe-pcnet.rom';
2029         } elsif ($device eq 'rtl8139') {
2030             $romfile = 'pxe-rtl8139.rom';
2031         }
2032         $tmpstr .= ",romfile=$romfile" if $romfile;
2033     }
2034
2035     return $tmpstr;
2036 }
2037
2038 sub print_netdev_full {
2039     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2040
2041     my $i = '';
2042     if ($netid =~ m/^net(\d+)$/) {
2043         $i = int($1);
2044     }
2045
2046     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2047
2048     my $ifname = "tap${vmid}i$i";
2049
2050     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2051     die "interface name '$ifname' is too long (max 15 character)\n"
2052         if length($ifname) >= 16;
2053
2054     my $vhostparam = '';
2055     if (is_native($arch)) {
2056         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2057     }
2058
2059     my $vmname = $conf->{name} || "vm$vmid";
2060
2061     my $netdev = "";
2062     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2063
2064     if ($net->{bridge}) {
2065         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2066     } else {
2067         $netdev = "type=user,id=$netid,hostname=$vmname";
2068     }
2069
2070     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2071
2072     return $netdev;
2073 }
2074
2075
2076 sub print_cpu_device {
2077     my ($conf, $id) = @_;
2078
2079     my $kvm = $conf->{kvm} // 1;
2080     my $cpu = $kvm ? "kvm64" : "qemu64";
2081     if (my $cputype = $conf->{cpu}) {
2082         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2083             or die "Cannot parse cpu description: $cputype\n";
2084         $cpu = $cpuconf->{cputype};
2085     }
2086
2087     my $cores = $conf->{cores} || 1;
2088
2089     my $current_core = ($id - 1) % $cores;
2090     my $current_socket = int(($id - 1 - $current_core)/$cores);
2091
2092     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2093 }
2094
2095 my $vga_map = {
2096     'cirrus' => 'cirrus-vga',
2097     'std' => 'VGA',
2098     'vmware' => 'vmware-svga',
2099     'virtio' => 'virtio-vga',
2100 };
2101
2102 sub print_vga_device {
2103     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2104
2105     my $type = $vga_map->{$vga->{type}};
2106     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2107         $type = 'virtio-gpu';
2108     }
2109     my $vgamem_mb = $vga->{memory};
2110     if ($qxlnum) {
2111         $type = $id ? 'qxl' : 'qxl-vga';
2112     }
2113     die "no devicetype for $vga->{type}\n" if !$type;
2114
2115     my $memory = "";
2116     if ($vgamem_mb) {
2117         if ($vga->{type} eq 'virtio') {
2118             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2119             $memory = ",max_hostmem=$bytes";
2120         } elsif ($qxlnum) {
2121             # from https://www.spice-space.org/multiple-monitors.html
2122             $memory = ",vgamem_mb=$vga->{memory}";
2123             my $ram = $vgamem_mb * 4;
2124             my $vram = $vgamem_mb * 2;
2125             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2126         } else {
2127             $memory = ",vgamem_mb=$vga->{memory}";
2128         }
2129     } elsif ($qxlnum && $id) {
2130         $memory = ",ram_size=67108864,vram_size=33554432";
2131     }
2132
2133     my $q35 = machine_type_is_q35($conf);
2134     my $vgaid = "vga" . ($id // '');
2135     my $pciaddr;
2136
2137     if ($q35 && $vgaid eq 'vga') {
2138         # the first display uses pcie.0 bus on q35 machines
2139         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2140     } else {
2141         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2142     }
2143
2144     return "$type,id=${vgaid}${memory}${pciaddr}";
2145 }
2146
2147 sub drive_is_cloudinit {
2148     my ($drive) = @_;
2149     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2150 }
2151
2152 sub drive_is_cdrom {
2153     my ($drive, $exclude_cloudinit) = @_;
2154
2155     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2156
2157     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2158
2159 }
2160
2161 sub parse_number_sets {
2162     my ($set) = @_;
2163     my $res = [];
2164     foreach my $part (split(/;/, $set)) {
2165         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2166             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2167             push @$res, [ $1, $2 ];
2168         } else {
2169             die "invalid range: $part\n";
2170         }
2171     }
2172     return $res;
2173 }
2174
2175 sub parse_numa {
2176     my ($data) = @_;
2177
2178     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2179     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2180     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2181     return $res;
2182 }
2183
2184 sub parse_hostpci {
2185     my ($value) = @_;
2186
2187     return undef if !$value;
2188
2189     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2190
2191     my @idlist = split(/;/, $res->{host});
2192     delete $res->{host};
2193     foreach my $id (@idlist) {
2194         if ($id =~ m/\./) { # full id 00:00.1
2195             push @{$res->{pciid}}, {
2196                 id => $id,
2197             };
2198         } else { # partial id 00:00
2199             $res->{pciid} = PVE::SysFSTools::lspci($id);
2200         }
2201     }
2202     return $res;
2203 }
2204
2205 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2206 sub parse_net {
2207     my ($data) = @_;
2208
2209     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2210     if ($@) {
2211         warn $@;
2212         return undef;
2213     }
2214     if (!defined($res->{macaddr})) {
2215         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2216         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2217     }
2218     return $res;
2219 }
2220
2221 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2222 sub parse_ipconfig {
2223     my ($data) = @_;
2224
2225     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2226     if ($@) {
2227         warn $@;
2228         return undef;
2229     }
2230
2231     if ($res->{gw} && !$res->{ip}) {
2232         warn 'gateway specified without specifying an IP address';
2233         return undef;
2234     }
2235     if ($res->{gw6} && !$res->{ip6}) {
2236         warn 'IPv6 gateway specified without specifying an IPv6 address';
2237         return undef;
2238     }
2239     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2240         warn 'gateway specified together with DHCP';
2241         return undef;
2242     }
2243     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2244         # gw6 + auto/dhcp
2245         warn "IPv6 gateway specified together with $res->{ip6} address";
2246         return undef;
2247     }
2248
2249     if (!$res->{ip} && !$res->{ip6}) {
2250         return { ip => 'dhcp', ip6 => 'dhcp' };
2251     }
2252
2253     return $res;
2254 }
2255
2256 sub print_net {
2257     my $net = shift;
2258
2259     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2260 }
2261
2262 sub add_random_macs {
2263     my ($settings) = @_;
2264
2265     foreach my $opt (keys %$settings) {
2266         next if $opt !~ m/^net(\d+)$/;
2267         my $net = parse_net($settings->{$opt});
2268         next if !$net;
2269         $settings->{$opt} = print_net($net);
2270     }
2271 }
2272
2273 sub vm_is_volid_owner {
2274     my ($storecfg, $vmid, $volid) = @_;
2275
2276     if ($volid !~  m|^/|) {
2277         my ($path, $owner);
2278         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2279         if ($owner && ($owner == $vmid)) {
2280             return 1;
2281         }
2282     }
2283
2284     return undef;
2285 }
2286
2287 sub split_flagged_list {
2288     my $text = shift || '';
2289     $text =~ s/[,;]/ /g;
2290     $text =~ s/^\s+//;
2291     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2292 }
2293
2294 sub join_flagged_list {
2295     my ($how, $lst) = @_;
2296     join $how, map { $lst->{$_} . $_ } keys %$lst;
2297 }
2298
2299 sub vmconfig_delete_pending_option {
2300     my ($conf, $key, $force) = @_;
2301
2302     delete $conf->{pending}->{$key};
2303     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2304     $pending_delete_hash->{$key} = $force ? '!' : '';
2305     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2306 }
2307
2308 sub vmconfig_undelete_pending_option {
2309     my ($conf, $key) = @_;
2310
2311     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2312     delete $pending_delete_hash->{$key};
2313
2314     if (%$pending_delete_hash) {
2315         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2316     } else {
2317         delete $conf->{pending}->{delete};
2318     }
2319 }
2320
2321 sub vmconfig_register_unused_drive {
2322     my ($storecfg, $vmid, $conf, $drive) = @_;
2323
2324     if (drive_is_cloudinit($drive)) {
2325         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2326         warn $@ if $@;
2327     } elsif (!drive_is_cdrom($drive)) {
2328         my $volid = $drive->{file};
2329         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2330             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2331         }
2332     }
2333 }
2334
2335 sub vmconfig_cleanup_pending {
2336     my ($conf) = @_;
2337
2338     # remove pending changes when nothing changed
2339     my $changes;
2340     foreach my $opt (keys %{$conf->{pending}}) {
2341         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2342             $changes = 1;
2343             delete $conf->{pending}->{$opt};
2344         }
2345     }
2346
2347     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2348     my $pending_delete_hash = {};
2349     while (my ($opt, $force) = each %$current_delete_hash) {
2350         if (defined($conf->{$opt})) {
2351             $pending_delete_hash->{$opt} = $force;
2352         } else {
2353             $changes = 1;
2354         }
2355     }
2356
2357     if (%$pending_delete_hash) {
2358         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2359     } else {
2360         delete $conf->{pending}->{delete};
2361     }
2362
2363     return $changes;
2364 }
2365
2366 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2367 my $smbios1_fmt = {
2368     uuid => {
2369         type => 'string',
2370         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2371         format_description => 'UUID',
2372         description => "Set SMBIOS1 UUID.",
2373         optional => 1,
2374     },
2375     version => {
2376         type => 'string',
2377         pattern => '\S+',
2378         format_description => 'string',
2379         description => "Set SMBIOS1 version.",
2380         optional => 1,
2381     },
2382     serial => {
2383         type => 'string',
2384         pattern => '\S+',
2385         format_description => 'string',
2386         description => "Set SMBIOS1 serial number.",
2387         optional => 1,
2388     },
2389     manufacturer => {
2390         type => 'string',
2391         pattern => '\S+',
2392         format_description => 'string',
2393         description => "Set SMBIOS1 manufacturer.",
2394         optional => 1,
2395     },
2396     product => {
2397         type => 'string',
2398         pattern => '\S+',
2399         format_description => 'string',
2400         description => "Set SMBIOS1 product ID.",
2401         optional => 1,
2402     },
2403     sku => {
2404         type => 'string',
2405         pattern => '\S+',
2406         format_description => 'string',
2407         description => "Set SMBIOS1 SKU string.",
2408         optional => 1,
2409     },
2410     family => {
2411         type => 'string',
2412         pattern => '\S+',
2413         format_description => 'string',
2414         description => "Set SMBIOS1 family string.",
2415         optional => 1,
2416     },
2417 };
2418
2419 sub parse_smbios1 {
2420     my ($data) = @_;
2421
2422     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2423     warn $@ if $@;
2424     return $res;
2425 }
2426
2427 sub print_smbios1 {
2428     my ($smbios1) = @_;
2429     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2430 }
2431
2432 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2433
2434 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2435 sub verify_bootdisk {
2436     my ($value, $noerr) = @_;
2437
2438     return $value if is_valid_drivename($value);
2439
2440     return undef if $noerr;
2441
2442     die "invalid boot disk '$value'\n";
2443 }
2444
2445 sub parse_watchdog {
2446     my ($value) = @_;
2447
2448     return undef if !$value;
2449
2450     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2451     warn $@ if $@;
2452     return $res;
2453 }
2454
2455 sub parse_guest_agent {
2456     my ($value) = @_;
2457
2458     return {} if !defined($value->{agent});
2459
2460     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2461     warn $@ if $@;
2462
2463     # if the agent is disabled ignore the other potentially set properties
2464     return {} if !$res->{enabled};
2465     return $res;
2466 }
2467
2468 sub parse_vga {
2469     my ($value) = @_;
2470
2471     return {} if !$value;
2472     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2473     warn $@ if $@;
2474     return $res;
2475 }
2476
2477 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2478 sub verify_usb_device {
2479     my ($value, $noerr) = @_;
2480
2481     return $value if parse_usb_device($value);
2482
2483     return undef if $noerr;
2484
2485     die "unable to parse usb device\n";
2486 }
2487
2488 # add JSON properties for create and set function
2489 sub json_config_properties {
2490     my $prop = shift;
2491
2492     foreach my $opt (keys %$confdesc) {
2493         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2494         $prop->{$opt} = $confdesc->{$opt};
2495     }
2496
2497     return $prop;
2498 }
2499
2500 # return copy of $confdesc_cloudinit to generate documentation
2501 sub cloudinit_config_properties {
2502
2503     return dclone($confdesc_cloudinit);
2504 }
2505
2506 sub check_type {
2507     my ($key, $value) = @_;
2508
2509     die "unknown setting '$key'\n" if !$confdesc->{$key};
2510
2511     my $type = $confdesc->{$key}->{type};
2512
2513     if (!defined($value)) {
2514         die "got undefined value\n";
2515     }
2516
2517     if ($value =~ m/[\n\r]/) {
2518         die "property contains a line feed\n";
2519     }
2520
2521     if ($type eq 'boolean') {
2522         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2523         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2524         die "type check ('boolean') failed - got '$value'\n";
2525     } elsif ($type eq 'integer') {
2526         return int($1) if $value =~ m/^(\d+)$/;
2527         die "type check ('integer') failed - got '$value'\n";
2528     } elsif ($type eq 'number') {
2529         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2530         die "type check ('number') failed - got '$value'\n";
2531     } elsif ($type eq 'string') {
2532         if (my $fmt = $confdesc->{$key}->{format}) {
2533             PVE::JSONSchema::check_format($fmt, $value);
2534             return $value;
2535         }
2536         $value =~ s/^\"(.*)\"$/$1/;
2537         return $value;
2538     } else {
2539         die "internal error"
2540     }
2541 }
2542
2543 sub touch_config {
2544     my ($vmid) = @_;
2545
2546     my $conf = PVE::QemuConfig->config_file($vmid);
2547     utime undef, undef, $conf;
2548 }
2549
2550 sub destroy_vm {
2551     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2552
2553     my $conffile = PVE::QemuConfig->config_file($vmid);
2554
2555     my $conf = PVE::QemuConfig->load_config($vmid);
2556
2557     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2558
2559     if ($conf->{template}) {
2560         # check if any base image is still used by a linked clone
2561         foreach_drive($conf, sub {
2562                 my ($ds, $drive) = @_;
2563
2564                 return if drive_is_cdrom($drive);
2565
2566                 my $volid = $drive->{file};
2567
2568                 return if !$volid || $volid =~ m|^/|;
2569
2570                 die "base volume '$volid' is still in use by linked cloned\n"
2571                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2572
2573         });
2574     }
2575
2576     # only remove disks owned by this VM
2577     foreach_drive($conf, sub {
2578         my ($ds, $drive) = @_;
2579
2580         return if drive_is_cdrom($drive, 1);
2581
2582         my $volid = $drive->{file};
2583
2584         return if !$volid || $volid =~ m|^/|;
2585
2586         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2587         return if !$path || !$owner || ($owner != $vmid);
2588
2589         eval {
2590             PVE::Storage::vdisk_free($storecfg, $volid);
2591         };
2592         warn "Could not remove disk '$volid', check manually: $@" if $@;
2593
2594     });
2595
2596     if ($keep_empty_config) {
2597         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2598     } else {
2599         unlink $conffile;
2600     }
2601
2602     # also remove unused disk
2603     eval {
2604         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2605
2606         eval {
2607             PVE::Storage::foreach_volid($dl, sub {
2608                 my ($volid, $sid, $volname, $d) = @_;
2609                 PVE::Storage::vdisk_free($storecfg, $volid);
2610             });
2611         };
2612         warn $@ if $@;
2613
2614     };
2615     warn $@ if $@;
2616 }
2617
2618 sub parse_vm_config {
2619     my ($filename, $raw) = @_;
2620
2621     return undef if !defined($raw);
2622
2623     my $res = {
2624         digest => Digest::SHA::sha1_hex($raw),
2625         snapshots => {},
2626         pending => {},
2627     };
2628
2629     $filename =~ m|/qemu-server/(\d+)\.conf$|
2630         || die "got strange filename '$filename'";
2631
2632     my $vmid = $1;
2633
2634     my $conf = $res;
2635     my $descr;
2636     my $section = '';
2637
2638     my @lines = split(/\n/, $raw);
2639     foreach my $line (@lines) {
2640         next if $line =~ m/^\s*$/;
2641
2642         if ($line =~ m/^\[PENDING\]\s*$/i) {
2643             $section = 'pending';
2644             if (defined($descr)) {
2645                 $descr =~ s/\s+$//;
2646                 $conf->{description} = $descr;
2647             }
2648             $descr = undef;
2649             $conf = $res->{$section} = {};
2650             next;
2651
2652         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2653             $section = $1;
2654             if (defined($descr)) {
2655                 $descr =~ s/\s+$//;
2656                 $conf->{description} = $descr;
2657             }
2658             $descr = undef;
2659             $conf = $res->{snapshots}->{$section} = {};
2660             next;
2661         }
2662
2663         if ($line =~ m/^\#(.*)\s*$/) {
2664             $descr = '' if !defined($descr);
2665             $descr .= PVE::Tools::decode_text($1) . "\n";
2666             next;
2667         }
2668
2669         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2670             $descr = '' if !defined($descr);
2671             $descr .= PVE::Tools::decode_text($2);
2672         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2673             $conf->{snapstate} = $1;
2674         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2675             my $key = $1;
2676             my $value = $2;
2677             $conf->{$key} = $value;
2678         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2679             my $value = $1;
2680             if ($section eq 'pending') {
2681                 $conf->{delete} = $value; # we parse this later
2682             } else {
2683                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2684             }
2685         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2686             my $key = $1;
2687             my $value = $2;
2688             eval { $value = check_type($key, $value); };
2689             if ($@) {
2690                 warn "vm $vmid - unable to parse value of '$key' - $@";
2691             } else {
2692                 $key = 'ide2' if $key eq 'cdrom';
2693                 my $fmt = $confdesc->{$key}->{format};
2694                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2695                     my $v = parse_drive($key, $value);
2696                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2697                         $v->{file} = $volid;
2698                         $value = print_drive($vmid, $v);
2699                     } else {
2700                         warn "vm $vmid - unable to parse value of '$key'\n";
2701                         next;
2702                     }
2703                 }
2704
2705                 $conf->{$key} = $value;
2706             }
2707         }
2708     }
2709
2710     if (defined($descr)) {
2711         $descr =~ s/\s+$//;
2712         $conf->{description} = $descr;
2713     }
2714     delete $res->{snapstate}; # just to be sure
2715
2716     return $res;
2717 }
2718
2719 sub write_vm_config {
2720     my ($filename, $conf) = @_;
2721
2722     delete $conf->{snapstate}; # just to be sure
2723
2724     if ($conf->{cdrom}) {
2725         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2726         $conf->{ide2} = $conf->{cdrom};
2727         delete $conf->{cdrom};
2728     }
2729
2730     # we do not use 'smp' any longer
2731     if ($conf->{sockets}) {
2732         delete $conf->{smp};
2733     } elsif ($conf->{smp}) {
2734         $conf->{sockets} = $conf->{smp};
2735         delete $conf->{cores};
2736         delete $conf->{smp};
2737     }
2738
2739     my $used_volids = {};
2740
2741     my $cleanup_config = sub {
2742         my ($cref, $pending, $snapname) = @_;
2743
2744         foreach my $key (keys %$cref) {
2745             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2746                 $key eq 'snapstate' || $key eq 'pending';
2747             my $value = $cref->{$key};
2748             if ($key eq 'delete') {
2749                 die "propertry 'delete' is only allowed in [PENDING]\n"
2750                     if !$pending;
2751                 # fixme: check syntax?
2752                 next;
2753             }
2754             eval { $value = check_type($key, $value); };
2755             die "unable to parse value of '$key' - $@" if $@;
2756
2757             $cref->{$key} = $value;
2758
2759             if (!$snapname && is_valid_drivename($key)) {
2760                 my $drive = parse_drive($key, $value);
2761                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2762             }
2763         }
2764     };
2765
2766     &$cleanup_config($conf);
2767
2768     &$cleanup_config($conf->{pending}, 1);
2769
2770     foreach my $snapname (keys %{$conf->{snapshots}}) {
2771         die "internal error" if $snapname eq 'pending';
2772         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2773     }
2774
2775     # remove 'unusedX' settings if we re-add a volume
2776     foreach my $key (keys %$conf) {
2777         my $value = $conf->{$key};
2778         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2779             delete $conf->{$key};
2780         }
2781     }
2782
2783     my $generate_raw_config = sub {
2784         my ($conf, $pending) = @_;
2785
2786         my $raw = '';
2787
2788         # add description as comment to top of file
2789         if (defined(my $descr = $conf->{description})) {
2790             if ($descr) {
2791                 foreach my $cl (split(/\n/, $descr)) {
2792                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2793                 }
2794             } else {
2795                 $raw .= "#\n" if $pending;
2796             }
2797         }
2798
2799         foreach my $key (sort keys %$conf) {
2800             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2801             $raw .= "$key: $conf->{$key}\n";
2802         }
2803         return $raw;
2804     };
2805
2806     my $raw = &$generate_raw_config($conf);
2807
2808     if (scalar(keys %{$conf->{pending}})){
2809         $raw .= "\n[PENDING]\n";
2810         $raw .= &$generate_raw_config($conf->{pending}, 1);
2811     }
2812
2813     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2814         $raw .= "\n[$snapname]\n";
2815         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2816     }
2817
2818     return $raw;
2819 }
2820
2821 sub load_defaults {
2822
2823     my $res = {};
2824
2825     # we use static defaults from our JSON schema configuration
2826     foreach my $key (keys %$confdesc) {
2827         if (defined(my $default = $confdesc->{$key}->{default})) {
2828             $res->{$key} = $default;
2829         }
2830     }
2831
2832     return $res;
2833 }
2834
2835 sub config_list {
2836     my $vmlist = PVE::Cluster::get_vmlist();
2837     my $res = {};
2838     return $res if !$vmlist || !$vmlist->{ids};
2839     my $ids = $vmlist->{ids};
2840
2841     foreach my $vmid (keys %$ids) {
2842         my $d = $ids->{$vmid};
2843         next if !$d->{node} || $d->{node} ne $nodename;
2844         next if !$d->{type} || $d->{type} ne 'qemu';
2845         $res->{$vmid}->{exists} = 1;
2846     }
2847     return $res;
2848 }
2849
2850 # test if VM uses local resources (to prevent migration)
2851 sub check_local_resources {
2852     my ($conf, $noerr) = @_;
2853
2854     my $loc_res = 0;
2855
2856     $loc_res = 1 if $conf->{hostusb}; # old syntax
2857     $loc_res = 1 if $conf->{hostpci}; # old syntax
2858
2859     $loc_res = 1 if $conf->{ivshmem};
2860
2861     foreach my $k (keys %$conf) {
2862         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2863         # sockets are safe: they will recreated be on the target side post-migrate
2864         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2865         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2866     }
2867
2868     die "VM uses local resources\n" if $loc_res && !$noerr;
2869
2870     return $loc_res;
2871 }
2872
2873 # check if used storages are available on all nodes (use by migrate)
2874 sub check_storage_availability {
2875     my ($storecfg, $conf, $node) = @_;
2876
2877     foreach_drive($conf, sub {
2878         my ($ds, $drive) = @_;
2879
2880         my $volid = $drive->{file};
2881         return if !$volid;
2882
2883         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2884         return if !$sid;
2885
2886         # check if storage is available on both nodes
2887         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2888         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2889    });
2890 }
2891
2892 # list nodes where all VM images are available (used by has_feature API)
2893 sub shared_nodes {
2894     my ($conf, $storecfg) = @_;
2895
2896     my $nodelist = PVE::Cluster::get_nodelist();
2897     my $nodehash = { map { $_ => 1 } @$nodelist };
2898     my $nodename = PVE::INotify::nodename();
2899
2900     foreach_drive($conf, sub {
2901         my ($ds, $drive) = @_;
2902
2903         my $volid = $drive->{file};
2904         return if !$volid;
2905
2906         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2907         if ($storeid) {
2908             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2909             if ($scfg->{disable}) {
2910                 $nodehash = {};
2911             } elsif (my $avail = $scfg->{nodes}) {
2912                 foreach my $node (keys %$nodehash) {
2913                     delete $nodehash->{$node} if !$avail->{$node};
2914                 }
2915             } elsif (!$scfg->{shared}) {
2916                 foreach my $node (keys %$nodehash) {
2917                     delete $nodehash->{$node} if $node ne $nodename
2918                 }
2919             }
2920         }
2921     });
2922
2923     return $nodehash
2924 }
2925
2926 sub check_cmdline {
2927     my ($pidfile, $pid) = @_;
2928
2929     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2930     if (defined($fh)) {
2931         my $line = <$fh>;
2932         $fh->close;
2933         return undef if !$line;
2934         my @param = split(/\0/, $line);
2935
2936         my $cmd = $param[0];
2937         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2938
2939         for (my $i = 0; $i < scalar (@param); $i++) {
2940             my $p = $param[$i];
2941             next if !$p;
2942             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2943                 my $p = $param[$i+1];
2944                 return 1 if $p && ($p eq $pidfile);
2945                 return undef;
2946             }
2947         }
2948     }
2949     return undef;
2950 }
2951
2952 sub check_running {
2953     my ($vmid, $nocheck, $node) = @_;
2954
2955     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2956
2957     die "unable to find configuration file for VM $vmid - no such machine\n"
2958         if !$nocheck && ! -f $filename;
2959
2960     my $pidfile = pidfile_name($vmid);
2961
2962     if (my $fd = IO::File->new("<$pidfile")) {
2963         my $st = stat($fd);
2964         my $line = <$fd>;
2965         close($fd);
2966
2967         my $mtime = $st->mtime;
2968         if ($mtime > time()) {
2969             warn "file '$filename' modified in future\n";
2970         }
2971
2972         if ($line =~ m/^(\d+)$/) {
2973             my $pid = $1;
2974             if (check_cmdline($pidfile, $pid)) {
2975                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2976                     return $pid;
2977                 }
2978             }
2979         }
2980     }
2981
2982     return undef;
2983 }
2984
2985 sub vzlist {
2986
2987     my $vzlist = config_list();
2988
2989     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2990
2991     while (defined(my $de = $fd->read)) {
2992         next if $de !~ m/^(\d+)\.pid$/;
2993         my $vmid = $1;
2994         next if !defined($vzlist->{$vmid});
2995         if (my $pid = check_running($vmid)) {
2996             $vzlist->{$vmid}->{pid} = $pid;
2997         }
2998     }
2999
3000     return $vzlist;
3001 }
3002
3003 sub disksize {
3004     my ($storecfg, $conf) = @_;
3005
3006     my $bootdisk = $conf->{bootdisk};
3007     return undef if !$bootdisk;
3008     return undef if !is_valid_drivename($bootdisk);
3009
3010     return undef if !$conf->{$bootdisk};
3011
3012     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3013     return undef if !defined($drive);
3014
3015     return undef if drive_is_cdrom($drive);
3016
3017     my $volid = $drive->{file};
3018     return undef if !$volid;
3019
3020     return $drive->{size};
3021 }
3022
3023 our $vmstatus_return_properties = {
3024     vmid => get_standard_option('pve-vmid'),
3025     status => {
3026         description => "Qemu process status.",
3027         type => 'string',
3028         enum => ['stopped', 'running'],
3029     },
3030     maxmem => {
3031         description => "Maximum memory in bytes.",
3032         type => 'integer',
3033         optional => 1,
3034         renderer => 'bytes',
3035     },
3036     maxdisk => {
3037         description => "Root disk size in bytes.",
3038         type => 'integer',
3039         optional => 1,
3040         renderer => 'bytes',
3041     },
3042     name => {
3043         description => "VM name.",
3044         type => 'string',
3045         optional => 1,
3046     },
3047     qmpstatus => {
3048         description => "Qemu QMP agent status.",
3049         type => 'string',
3050         optional => 1,
3051     },
3052     pid => {
3053         description => "PID of running qemu process.",
3054         type => 'integer',
3055         optional => 1,
3056     },
3057     uptime => {
3058         description => "Uptime.",
3059         type => 'integer',
3060         optional => 1,
3061         renderer => 'duration',
3062     },
3063     cpus => {
3064         description => "Maximum usable CPUs.",
3065         type => 'number',
3066         optional => 1,
3067     },
3068     lock => {
3069         description => "The current config lock, if any.",
3070         type => 'string',
3071         optional => 1,
3072     }
3073 };
3074
3075 my $last_proc_pid_stat;
3076
3077 # get VM status information
3078 # This must be fast and should not block ($full == false)
3079 # We only query KVM using QMP if $full == true (this can be slow)
3080 sub vmstatus {
3081     my ($opt_vmid, $full) = @_;
3082
3083     my $res = {};
3084
3085     my $storecfg = PVE::Storage::config();
3086
3087     my $list = vzlist();
3088     my $defaults = load_defaults();
3089
3090     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3091
3092     my $cpucount = $cpuinfo->{cpus} || 1;
3093
3094     foreach my $vmid (keys %$list) {
3095         next if $opt_vmid && ($vmid ne $opt_vmid);
3096
3097         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3098         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3099
3100         my $d = { vmid => $vmid };
3101         $d->{pid} = $list->{$vmid}->{pid};
3102
3103         # fixme: better status?
3104         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3105
3106         my $size = disksize($storecfg, $conf);
3107         if (defined($size)) {
3108             $d->{disk} = 0; # no info available
3109             $d->{maxdisk} = $size;
3110         } else {
3111             $d->{disk} = 0;
3112             $d->{maxdisk} = 0;
3113         }
3114
3115         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3116             * ($conf->{cores} || $defaults->{cores});
3117         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3118         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3119
3120         $d->{name} = $conf->{name} || "VM $vmid";
3121         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3122             : $defaults->{memory}*(1024*1024);
3123
3124         if ($conf->{balloon}) {
3125             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3126             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3127                 : $defaults->{shares};
3128         }
3129
3130         $d->{uptime} = 0;
3131         $d->{cpu} = 0;
3132         $d->{mem} = 0;
3133
3134         $d->{netout} = 0;
3135         $d->{netin} = 0;
3136
3137         $d->{diskread} = 0;
3138         $d->{diskwrite} = 0;
3139
3140         $d->{template} = PVE::QemuConfig->is_template($conf);
3141
3142         $d->{serial} = 1 if conf_has_serial($conf);
3143         $d->{lock} = $conf->{lock} if $conf->{lock};
3144
3145         $res->{$vmid} = $d;
3146     }
3147
3148     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3149     foreach my $dev (keys %$netdev) {
3150         next if $dev !~ m/^tap([1-9]\d*)i/;
3151         my $vmid = $1;
3152         my $d = $res->{$vmid};
3153         next if !$d;
3154
3155         $d->{netout} += $netdev->{$dev}->{receive};
3156         $d->{netin} += $netdev->{$dev}->{transmit};
3157
3158         if ($full) {
3159             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3160             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3161         }
3162
3163     }
3164
3165     my $ctime = gettimeofday;
3166
3167     foreach my $vmid (keys %$list) {
3168
3169         my $d = $res->{$vmid};
3170         my $pid = $d->{pid};
3171         next if !$pid;
3172
3173         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3174         next if !$pstat; # not running
3175
3176         my $used = $pstat->{utime} + $pstat->{stime};
3177
3178         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3179
3180         if ($pstat->{vsize}) {
3181             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3182         }
3183
3184         my $old = $last_proc_pid_stat->{$pid};
3185         if (!$old) {
3186             $last_proc_pid_stat->{$pid} = {
3187                 time => $ctime,
3188                 used => $used,
3189                 cpu => 0,
3190             };
3191             next;
3192         }
3193
3194         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3195
3196         if ($dtime > 1000) {
3197             my $dutime = $used -  $old->{used};
3198
3199             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3200             $last_proc_pid_stat->{$pid} = {
3201                 time => $ctime,
3202                 used => $used,
3203                 cpu => $d->{cpu},
3204             };
3205         } else {
3206             $d->{cpu} = $old->{cpu};
3207         }
3208     }
3209
3210     return $res if !$full;
3211
3212     my $qmpclient = PVE::QMPClient->new();
3213
3214     my $ballooncb = sub {
3215         my ($vmid, $resp) = @_;
3216
3217         my $info = $resp->{'return'};
3218         return if !$info->{max_mem};
3219
3220         my $d = $res->{$vmid};
3221
3222         # use memory assigned to VM
3223         $d->{maxmem} = $info->{max_mem};
3224         $d->{balloon} = $info->{actual};
3225
3226         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3227             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3228             $d->{freemem} = $info->{free_mem};
3229         }
3230
3231         $d->{ballooninfo} = $info;
3232     };
3233
3234     my $blockstatscb = sub {
3235         my ($vmid, $resp) = @_;
3236         my $data = $resp->{'return'} || [];
3237         my $totalrdbytes = 0;
3238         my $totalwrbytes = 0;
3239
3240         for my $blockstat (@$data) {
3241             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3242             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3243
3244             $blockstat->{device} =~ s/drive-//;
3245             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3246         }
3247         $res->{$vmid}->{diskread} = $totalrdbytes;
3248         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3249     };
3250
3251     my $statuscb = sub {
3252         my ($vmid, $resp) = @_;
3253
3254         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3255         # this fails if ballon driver is not loaded, so this must be
3256         # the last commnand (following command are aborted if this fails).
3257         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3258
3259         my $status = 'unknown';
3260         if (!defined($status = $resp->{'return'}->{status})) {
3261             warn "unable to get VM status\n";
3262             return;
3263         }
3264
3265         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3266     };
3267
3268     foreach my $vmid (keys %$list) {
3269         next if $opt_vmid && ($vmid ne $opt_vmid);
3270         next if !$res->{$vmid}->{pid}; # not running
3271         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3272     }
3273
3274     $qmpclient->queue_execute(undef, 2);
3275
3276     foreach my $vmid (keys %$list) {
3277         next if $opt_vmid && ($vmid ne $opt_vmid);
3278         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3279     }
3280
3281     return $res;
3282 }
3283
3284 sub foreach_drive {
3285     my ($conf, $func, @param) = @_;
3286
3287     foreach my $ds (valid_drive_names()) {
3288         next if !defined($conf->{$ds});
3289
3290         my $drive = parse_drive($ds, $conf->{$ds});
3291         next if !$drive;
3292
3293         &$func($ds, $drive, @param);
3294     }
3295 }
3296
3297 sub foreach_volid {
3298     my ($conf, $func, @param) = @_;
3299
3300     my $volhash = {};
3301
3302     my $test_volid = sub {
3303         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3304
3305         return if !$volid;
3306
3307         $volhash->{$volid}->{cdrom} //= 1;
3308         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3309
3310         $volhash->{$volid}->{replicate} //= 0;
3311         $volhash->{$volid}->{replicate} = 1 if $replicate;
3312
3313         $volhash->{$volid}->{shared} //= 0;
3314         $volhash->{$volid}->{shared} = 1 if $shared;
3315
3316         $volhash->{$volid}->{referenced_in_config} //= 0;
3317         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3318
3319         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3320             if defined($snapname);
3321     };
3322
3323     foreach_drive($conf, sub {
3324         my ($ds, $drive) = @_;
3325         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3326     });
3327
3328     foreach my $snapname (keys %{$conf->{snapshots}}) {
3329         my $snap = $conf->{snapshots}->{$snapname};
3330         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3331         foreach_drive($snap, sub {
3332             my ($ds, $drive) = @_;
3333             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3334         });
3335     }
3336
3337     foreach my $volid (keys %$volhash) {
3338         &$func($volid, $volhash->{$volid}, @param);
3339     }
3340 }
3341
3342 sub conf_has_serial {
3343     my ($conf) = @_;
3344
3345     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3346         if ($conf->{"serial$i"}) {
3347             return 1;
3348         }
3349     }
3350
3351     return 0;
3352 }
3353
3354 sub vga_conf_has_spice {
3355     my ($vga) = @_;
3356
3357     my $vgaconf = parse_vga($vga);
3358     my $vgatype = $vgaconf->{type};
3359     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3360
3361     return $1 || 1;
3362 }
3363
3364 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3365 sub get_host_arch() {
3366     $host_arch = (POSIX::uname())[4] if !$host_arch;
3367     return $host_arch;
3368 }
3369
3370 sub is_native($) {
3371     my ($arch) = @_;
3372     return get_host_arch() eq $arch;
3373 }
3374
3375 my $default_machines = {
3376     x86_64 => 'pc',
3377     aarch64 => 'virt',
3378 };
3379
3380 sub get_basic_machine_info {
3381     my ($conf, $forcemachine) = @_;
3382
3383     my $arch = $conf->{arch} // get_host_arch();
3384     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3385     return ($arch, $machine);
3386 }
3387
3388 sub get_ovmf_files($) {
3389     my ($arch) = @_;
3390
3391     my $ovmf = $OVMF->{$arch}
3392         or die "no OVMF images known for architecture '$arch'\n";
3393
3394     return @$ovmf;
3395 }
3396
3397 my $Arch2Qemu = {
3398     aarch64 => '/usr/bin/qemu-system-aarch64',
3399     x86_64 => '/usr/bin/qemu-system-x86_64',
3400 };
3401 sub get_command_for_arch($) {
3402     my ($arch) = @_;
3403     return '/usr/bin/kvm' if is_native($arch);
3404
3405     my $cmd = $Arch2Qemu->{$arch}
3406         or die "don't know how to emulate architecture '$arch'\n";
3407     return $cmd;
3408 }
3409
3410 sub get_cpu_options {
3411     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3412
3413     my $cpuFlags = [];
3414     my $ostype = $conf->{ostype};
3415
3416     my $cpu = $kvm ? "kvm64" : "qemu64";
3417     if ($arch eq 'aarch64') {
3418         $cpu = 'cortex-a57';
3419     }
3420     my $hv_vendor_id;
3421     if (my $cputype = $conf->{cpu}) {
3422         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3423             or die "Cannot parse cpu description: $cputype\n";
3424         $cpu = $cpuconf->{cputype};
3425         $kvm_off = 1 if $cpuconf->{hidden};
3426         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3427
3428         if (defined(my $flags = $cpuconf->{flags})) {
3429             push @$cpuFlags, split(";", $flags);
3430         }
3431     }
3432
3433     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3434
3435     push @$cpuFlags , '-x2apic'
3436         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3437
3438     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3439
3440     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3441
3442     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3443
3444         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3445         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3446     }
3447
3448     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3449
3450     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3451
3452     push @$cpuFlags, 'kvm=off' if $kvm_off;
3453
3454     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3455         push @$cpuFlags, "vendor=${cpu_vendor}"
3456             if $cpu_vendor ne 'default';
3457     } elsif ($arch ne 'aarch64') {
3458         die "internal error"; # should not happen
3459     }
3460
3461     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3462
3463     return ('-cpu', $cpu);
3464 }
3465
3466 sub config_to_command {
3467     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3468
3469     my $cmd = [];
3470     my $globalFlags = [];
3471     my $machineFlags = [];
3472     my $rtcFlags = [];
3473     my $devices = [];
3474     my $pciaddr = '';
3475     my $bridges = {};
3476     my $kvmver = kvm_user_version();
3477     my $vernum = 0; # unknown
3478     my $ostype = $conf->{ostype};
3479     my $winversion = windows_version($ostype);
3480     my $kvm = $conf->{kvm};
3481
3482     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3483     $kvm //= 1 if is_native($arch);
3484
3485     if ($kvm) {
3486         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3487             if !defined kvm_version();
3488     }
3489
3490     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3491         $vernum = $1*1000000+$2*1000;
3492     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3493         $vernum = $1*1000000+$2*1000+$3;
3494     }
3495
3496     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3497
3498     my $have_ovz = -f '/proc/vz/vestat';
3499
3500     my $q35 = machine_type_is_q35($conf);
3501     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3502     my $use_old_bios_files = undef;
3503     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3504
3505     my $cpuunits = defined($conf->{cpuunits}) ?
3506             $conf->{cpuunits} : $defaults->{cpuunits};
3507
3508     push @$cmd, get_command_for_arch($arch);
3509
3510     push @$cmd, '-id', $vmid;
3511
3512     my $vmname = $conf->{name} || "vm$vmid";
3513
3514     push @$cmd, '-name', $vmname;
3515
3516     my $use_virtio = 0;
3517
3518     my $qmpsocket = qmp_socket($vmid);
3519     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3520     push @$cmd, '-mon', "chardev=qmp,mode=control";
3521
3522     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3523         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3524         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3525     }
3526
3527     push @$cmd, '-pidfile' , pidfile_name($vmid);
3528
3529     push @$cmd, '-daemonize';
3530
3531     if ($conf->{smbios1}) {
3532         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3533     }
3534
3535     if ($conf->{vmgenid}) {
3536         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3537     }
3538
3539     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3540     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3541         die "uefi base image not found\n" if ! -f $ovmf_code;
3542
3543         my $path;
3544         my $format;
3545         if (my $efidisk = $conf->{efidisk0}) {
3546             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3547             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3548             $format = $d->{format};
3549             if ($storeid) {
3550                 $path = PVE::Storage::path($storecfg, $d->{file});
3551                 if (!defined($format)) {
3552                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3553                     $format = qemu_img_format($scfg, $volname);
3554                 }
3555             } else {
3556                 $path = $d->{file};
3557                 die "efidisk format must be specified\n"
3558                     if !defined($format);
3559             }
3560         } else {
3561             warn "no efidisk configured! Using temporary efivars disk.\n";
3562             $path = "/tmp/$vmid-ovmf.fd";
3563             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3564             $format = 'raw';
3565         }
3566
3567         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3568         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3569     }
3570
3571
3572     # add usb controllers
3573     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3574     push @$devices, @usbcontrollers if @usbcontrollers;
3575     my $vga = parse_vga($conf->{vga});
3576
3577     my $qxlnum = vga_conf_has_spice($conf->{vga});
3578     $vga->{type} = 'qxl' if $qxlnum;
3579
3580     if (!$vga->{type}) {
3581         if ($arch eq 'aarch64') {
3582             $vga->{type} = 'virtio';
3583         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3584             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3585         } else {
3586             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3587         }
3588     }
3589
3590     # enable absolute mouse coordinates (needed by vnc)
3591     my $tablet;
3592     if (defined($conf->{tablet})) {
3593         $tablet = $conf->{tablet};
3594     } else {
3595         $tablet = $defaults->{tablet};
3596         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3597         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3598     }
3599
3600     if ($tablet) {
3601         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3602         my $kbd = print_keyboarddevice_full($conf, $arch);
3603         push @$devices, '-device', $kbd if defined($kbd);
3604     }
3605
3606     my $kvm_off = 0;
3607     my $gpu_passthrough;
3608
3609     # host pci devices
3610     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3611         my $d = parse_hostpci($conf->{"hostpci$i"});
3612         next if !$d;
3613
3614         my $pcie = $d->{pcie};
3615         if ($pcie) {
3616             die "q35 machine model is not enabled" if !$q35;
3617             # win7 wants to have the pcie devices directly on the pcie bus
3618             # instead of in the root port
3619             if ($winversion == 7) {
3620                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3621             } else {
3622                 $pciaddr = print_pcie_addr("hostpci$i");
3623             }
3624         } else {
3625             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3626         }
3627
3628         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3629         my $romfile = $d->{romfile};
3630
3631         my $xvga = '';
3632         if ($d->{'x-vga'}) {
3633             $xvga = ',x-vga=on';
3634             $kvm_off = 1;
3635             $vga->{type} = 'none' if !defined($conf->{vga});
3636             $gpu_passthrough = 1;
3637
3638             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3639                 $xvga = "";
3640             }
3641         }
3642         my $pcidevices = $d->{pciid};
3643         my $multifunction = 1 if @$pcidevices > 1;
3644         my $sysfspath;
3645         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3646             my $id = $pcidevices->[0]->{id};
3647             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3648             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3649         } elsif ($d->{mdev}) {
3650             warn "ignoring mediated device with multifunction device\n";
3651         }
3652
3653         my $j=0;
3654         foreach my $pcidevice (@$pcidevices) {
3655
3656             my $id = "hostpci$i";
3657             $id .= ".$j" if $multifunction;
3658             my $addr = $pciaddr;
3659             $addr .= ".$j" if $multifunction;
3660             my $devicestr = "vfio-pci";
3661             if ($sysfspath) {
3662                 $devicestr .= ",sysfsdev=$sysfspath";
3663             } else {
3664                 $devicestr .= ",host=$pcidevice->{id}";
3665             }
3666             $devicestr .= ",id=$id$addr";
3667
3668             if($j == 0){
3669                 $devicestr .= "$rombar$xvga";
3670                 $devicestr .= ",multifunction=on" if $multifunction;
3671                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3672             }
3673
3674             push @$devices, '-device', $devicestr;
3675             $j++;
3676         }
3677     }
3678
3679     # usb devices
3680     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3681     push @$devices, @usbdevices if @usbdevices;
3682     # serial devices
3683     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3684         if (my $path = $conf->{"serial$i"}) {
3685             if ($path eq 'socket') {
3686                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3687                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3688                 # On aarch64, serial0 is the UART device. Qemu only allows
3689                 # connecting UART devices via the '-serial' command line, as
3690                 # the device has a fixed slot on the hardware...
3691                 if ($arch eq 'aarch64' && $i == 0) {
3692                     push @$devices, '-serial', "chardev:serial$i";
3693                 } else {
3694                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3695                 }
3696             } else {
3697                 die "no such serial device\n" if ! -c $path;
3698                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3699                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3700             }
3701         }
3702     }
3703
3704     # parallel devices
3705     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3706         if (my $path = $conf->{"parallel$i"}) {
3707             die "no such parallel device\n" if ! -c $path;
3708             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3709             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3710             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3711         }
3712     }
3713
3714
3715     my $sockets = 1;
3716     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3717     $sockets = $conf->{sockets} if  $conf->{sockets};
3718
3719     my $cores = $conf->{cores} || 1;
3720
3721     my $maxcpus = $sockets * $cores;
3722
3723     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3724
3725     my $allowed_vcpus = $cpuinfo->{cpus};
3726
3727     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3728         if ($allowed_vcpus < $maxcpus);
3729
3730     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3731
3732         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3733         for (my $i = 2; $i <= $vcpus; $i++)  {
3734             my $cpustr = print_cpu_device($conf,$i);
3735             push @$cmd, '-device', $cpustr;
3736         }
3737
3738     } else {
3739
3740         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3741     }
3742     push @$cmd, '-nodefaults';
3743
3744     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3745
3746     my $bootindex_hash = {};
3747     my $i = 1;
3748     foreach my $o (split(//, $bootorder)) {
3749         $bootindex_hash->{$o} = $i*100;
3750         $i++;
3751     }
3752
3753     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3754
3755     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3756
3757     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3758
3759     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3760         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3761         my $socket = vnc_socket($vmid);
3762         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3763     } else {
3764         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3765         push @$cmd, '-nographic';
3766     }
3767
3768     # time drift fix
3769     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3770
3771     my $useLocaltime = $conf->{localtime};
3772
3773     if ($winversion >= 5) { # windows
3774         $useLocaltime = 1 if !defined($conf->{localtime});
3775
3776         # use time drift fix when acpi is enabled
3777         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3778             $tdf = 1 if !defined($conf->{tdf});
3779         }
3780     }
3781
3782     if ($winversion >= 6) {
3783         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3784         push @$cmd, '-no-hpet';
3785     }
3786
3787     push @$rtcFlags, 'driftfix=slew' if $tdf;
3788
3789     if (!$kvm) {
3790         push @$machineFlags, 'accel=tcg';
3791     }
3792
3793     if ($machine_type) {
3794         push @$machineFlags, "type=${machine_type}";
3795     }
3796
3797     if ($conf->{startdate}) {
3798         push @$rtcFlags, "base=$conf->{startdate}";
3799     } elsif ($useLocaltime) {
3800         push @$rtcFlags, 'base=localtime';
3801     }
3802
3803     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3804
3805     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3806
3807     push @$cmd, '-S' if $conf->{freeze};
3808
3809     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3810
3811     # enable sound
3812     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3813     #push @$cmd, '-soundhw', 'es1370';
3814     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3815
3816     if (parse_guest_agent($conf)->{enabled}) {
3817         my $qgasocket = qmp_socket($vmid, 1);
3818         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3819         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3820         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3821         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3822     }
3823
3824     my $spice_port;
3825
3826     if ($qxlnum) {
3827         if ($qxlnum > 1) {
3828             if ($winversion){
3829                 for(my $i = 1; $i < $qxlnum; $i++){
3830                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3831                 }
3832             } else {
3833                 # assume other OS works like Linux
3834                 my ($ram, $vram) = ("134217728", "67108864");
3835                 if ($vga->{memory}) {
3836                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3837                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3838                 }
3839                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3840                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3841             }
3842         }
3843
3844         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3845
3846         my $nodename = PVE::INotify::nodename();
3847         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3848         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3849         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3850         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3851         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3852
3853         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3854
3855         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3856         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3857         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3858     }
3859
3860     # enable balloon by default, unless explicitly disabled
3861     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3862         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3863         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3864     }
3865
3866     if ($conf->{watchdog}) {
3867         my $wdopts = parse_watchdog($conf->{watchdog});
3868         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3869         my $watchdog = $wdopts->{model} || 'i6300esb';
3870         push @$devices, '-device', "$watchdog$pciaddr";
3871         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3872     }
3873
3874     my $vollist = [];
3875     my $scsicontroller = {};
3876     my $ahcicontroller = {};
3877     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3878
3879     # Add iscsi initiator name if available
3880     if (my $initiator = get_initiator_name()) {
3881         push @$devices, '-iscsi', "initiator-name=$initiator";
3882     }
3883
3884     foreach_drive($conf, sub {
3885         my ($ds, $drive) = @_;
3886
3887         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3888             push @$vollist, $drive->{file};
3889         }
3890
3891         # ignore efidisk here, already added in bios/fw handling code above
3892         return if $drive->{interface} eq 'efidisk';
3893
3894         $use_virtio = 1 if $ds =~ m/^virtio/;
3895
3896         if (drive_is_cdrom ($drive)) {
3897             if ($bootindex_hash->{d}) {
3898                 $drive->{bootindex} = $bootindex_hash->{d};
3899                 $bootindex_hash->{d} += 1;
3900             }
3901         } else {
3902             if ($bootindex_hash->{c}) {
3903                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3904                 $bootindex_hash->{c} += 1;
3905             }
3906         }
3907
3908         if($drive->{interface} eq 'virtio'){
3909            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3910         }
3911
3912         if ($drive->{interface} eq 'scsi') {
3913
3914             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3915
3916             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3917             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3918
3919             my $iothread = '';
3920             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3921                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3922                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3923             } elsif ($drive->{iothread}) {
3924                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3925             }
3926
3927             my $queues = '';
3928             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3929                 $queues = ",num_queues=$drive->{queues}";
3930             }
3931
3932             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3933             $scsicontroller->{$controller}=1;
3934         }
3935
3936         if ($drive->{interface} eq 'sata') {
3937            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3938            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3939            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3940            $ahcicontroller->{$controller}=1;
3941         }
3942
3943         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3944         push @$devices, '-drive',$drive_cmd;
3945         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3946     });
3947
3948     for (my $i = 0; $i < $MAX_NETS; $i++) {
3949          next if !$conf->{"net$i"};
3950          my $d = parse_net($conf->{"net$i"});
3951          next if !$d;
3952
3953          $use_virtio = 1 if $d->{model} eq 'virtio';
3954
3955          if ($bootindex_hash->{n}) {
3956             $d->{bootindex} = $bootindex_hash->{n};
3957             $bootindex_hash->{n} += 1;
3958          }
3959
3960          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3961          push @$devices, '-netdev', $netdevfull;
3962
3963          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3964          push @$devices, '-device', $netdevicefull;
3965     }
3966
3967     if ($conf->{ivshmem}) {
3968         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3969
3970         my $bus;
3971         if ($q35) {
3972             $bus = print_pcie_addr("ivshmem");
3973         } else {
3974             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3975         }
3976
3977         my $ivshmem_name = $ivshmem->{name} // $vmid;
3978         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3979
3980         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3981         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3982     }
3983
3984     if (!$q35) {
3985         # add pci bridges
3986         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3987            $bridges->{1} = 1;
3988            $bridges->{2} = 1;
3989         }
3990
3991         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3992
3993         while (my ($k, $v) = each %$bridges) {
3994             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3995             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3996         }
3997     }
3998
3999     push @$cmd, @$devices;
4000     push @$cmd, '-rtc', join(',', @$rtcFlags)
4001         if scalar(@$rtcFlags);
4002     push @$cmd, '-machine', join(',', @$machineFlags)
4003         if scalar(@$machineFlags);
4004     push @$cmd, '-global', join(',', @$globalFlags)
4005         if scalar(@$globalFlags);
4006
4007     if (my $vmstate = $conf->{vmstate}) {
4008         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4009         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4010         push @$cmd, '-loadstate', $statepath;
4011     }
4012
4013     # add custom args
4014     if ($conf->{args}) {
4015         my $aa = PVE::Tools::split_args($conf->{args});
4016         push @$cmd, @$aa;
4017     }
4018
4019     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4020 }
4021
4022 sub vnc_socket {
4023     my ($vmid) = @_;
4024     return "${var_run_tmpdir}/$vmid.vnc";
4025 }
4026
4027 sub spice_port {
4028     my ($vmid) = @_;
4029
4030     my $res = vm_mon_cmd($vmid, 'query-spice');
4031
4032     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4033 }
4034
4035 sub qmp_socket {
4036     my ($vmid, $qga, $name) = @_;
4037     my $sockettype = $qga ? 'qga' : 'qmp';
4038     my $ext = $name ? '-'.$name : '';
4039     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4040 }
4041
4042 sub pidfile_name {
4043     my ($vmid) = @_;
4044     return "${var_run_tmpdir}/$vmid.pid";
4045 }
4046
4047 sub vm_devices_list {
4048     my ($vmid) = @_;
4049
4050     my $res = vm_mon_cmd($vmid, 'query-pci');
4051     my $devices_to_check = [];
4052     my $devices = {};
4053     foreach my $pcibus (@$res) {
4054         push @$devices_to_check, @{$pcibus->{devices}},
4055     }
4056
4057     while (@$devices_to_check) {
4058         my $to_check = [];
4059         for my $d (@$devices_to_check) {
4060             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4061             next if !$d->{'pci_bridge'};
4062
4063             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4064             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4065         }
4066         $devices_to_check = $to_check;
4067     }
4068
4069     my $resblock = vm_mon_cmd($vmid, 'query-block');
4070     foreach my $block (@$resblock) {
4071         if($block->{device} =~ m/^drive-(\S+)/){
4072                 $devices->{$1} = 1;
4073         }
4074     }
4075
4076     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4077     foreach my $mice (@$resmice) {
4078         if ($mice->{name} eq 'QEMU HID Tablet') {
4079             $devices->{tablet} = 1;
4080             last;
4081         }
4082     }
4083
4084     # for usb devices there is no query-usb
4085     # but we can iterate over the entries in
4086     # qom-list path=/machine/peripheral
4087     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4088     foreach my $per (@$resperipheral) {
4089         if ($per->{name} =~ m/^usb\d+$/) {
4090             $devices->{$per->{name}} = 1;
4091         }
4092     }
4093
4094     return $devices;
4095 }
4096
4097 sub vm_deviceplug {
4098     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4099
4100     my $q35 = machine_type_is_q35($conf);
4101
4102     my $devices_list = vm_devices_list($vmid);
4103     return 1 if defined($devices_list->{$deviceid});
4104
4105     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4106
4107     if ($deviceid eq 'tablet') {
4108
4109         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4110
4111     } elsif ($deviceid eq 'keyboard') {
4112
4113         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4114
4115     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4116
4117         die "usb hotplug currently not reliable\n";
4118         # since we can't reliably hot unplug all added usb devices
4119         # and usb passthrough disables live migration
4120         # we disable usb hotplugging for now
4121         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4122
4123     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4124
4125         qemu_iothread_add($vmid, $deviceid, $device);
4126
4127         qemu_driveadd($storecfg, $vmid, $device);
4128         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4129
4130         qemu_deviceadd($vmid, $devicefull);
4131         eval { qemu_deviceaddverify($vmid, $deviceid); };
4132         if (my $err = $@) {
4133             eval { qemu_drivedel($vmid, $deviceid); };
4134             warn $@ if $@;
4135             die $err;
4136         }
4137
4138     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4139
4140
4141         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4142         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4143         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4144
4145         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4146
4147         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4148             qemu_iothread_add($vmid, $deviceid, $device);
4149             $devicefull .= ",iothread=iothread-$deviceid";
4150         }
4151
4152         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4153             $devicefull .= ",num_queues=$device->{queues}";
4154         }
4155
4156         qemu_deviceadd($vmid, $devicefull);
4157         qemu_deviceaddverify($vmid, $deviceid);
4158
4159     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4160
4161         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4162         qemu_driveadd($storecfg, $vmid, $device);
4163
4164         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4165         eval { qemu_deviceadd($vmid, $devicefull); };
4166         if (my $err = $@) {
4167             eval { qemu_drivedel($vmid, $deviceid); };
4168             warn $@ if $@;
4169             die $err;
4170         }
4171
4172     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4173
4174         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4175
4176         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4177         my $use_old_bios_files = undef;
4178         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4179
4180         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4181         qemu_deviceadd($vmid, $netdevicefull);
4182         eval {
4183             qemu_deviceaddverify($vmid, $deviceid);
4184             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4185         };
4186         if (my $err = $@) {
4187             eval { qemu_netdevdel($vmid, $deviceid); };
4188             warn $@ if $@;
4189             die $err;
4190         }
4191
4192     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4193
4194         my $bridgeid = $2;
4195         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4196         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4197
4198         qemu_deviceadd($vmid, $devicefull);
4199         qemu_deviceaddverify($vmid, $deviceid);
4200
4201     } else {
4202         die "can't hotplug device '$deviceid'\n";
4203     }
4204
4205     return 1;
4206 }
4207
4208 # fixme: this should raise exceptions on error!
4209 sub vm_deviceunplug {
4210     my ($vmid, $conf, $deviceid) = @_;
4211
4212     my $devices_list = vm_devices_list($vmid);
4213     return 1 if !defined($devices_list->{$deviceid});
4214
4215     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4216
4217     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4218
4219         qemu_devicedel($vmid, $deviceid);
4220
4221     } elsif ($deviceid =~ m/^usb\d+$/) {
4222
4223         die "usb hotplug currently not reliable\n";
4224         # when unplugging usb devices this way,
4225         # there may be remaining usb controllers/hubs
4226         # so we disable it for now
4227         qemu_devicedel($vmid, $deviceid);
4228         qemu_devicedelverify($vmid, $deviceid);
4229
4230     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4231
4232         qemu_devicedel($vmid, $deviceid);
4233         qemu_devicedelverify($vmid, $deviceid);
4234         qemu_drivedel($vmid, $deviceid);
4235         qemu_iothread_del($conf, $vmid, $deviceid);
4236
4237     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4238
4239         qemu_devicedel($vmid, $deviceid);
4240         qemu_devicedelverify($vmid, $deviceid);
4241         qemu_iothread_del($conf, $vmid, $deviceid);
4242
4243     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4244
4245         qemu_devicedel($vmid, $deviceid);
4246         qemu_drivedel($vmid, $deviceid);
4247         qemu_deletescsihw($conf, $vmid, $deviceid);
4248
4249     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4250
4251         qemu_devicedel($vmid, $deviceid);
4252         qemu_devicedelverify($vmid, $deviceid);
4253         qemu_netdevdel($vmid, $deviceid);
4254
4255     } else {
4256         die "can't unplug device '$deviceid'\n";
4257     }
4258
4259     return 1;
4260 }
4261
4262 sub qemu_deviceadd {
4263     my ($vmid, $devicefull) = @_;
4264
4265     $devicefull = "driver=".$devicefull;
4266     my %options =  split(/[=,]/, $devicefull);
4267
4268     vm_mon_cmd($vmid, "device_add" , %options);
4269 }
4270
4271 sub qemu_devicedel {
4272     my ($vmid, $deviceid) = @_;
4273
4274     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4275 }
4276
4277 sub qemu_iothread_add {
4278     my($vmid, $deviceid, $device) = @_;
4279
4280     if ($device->{iothread}) {
4281         my $iothreads = vm_iothreads_list($vmid);
4282         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4283     }
4284 }
4285
4286 sub qemu_iothread_del {
4287     my($conf, $vmid, $deviceid) = @_;
4288
4289     my $confid = $deviceid;
4290     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4291         $confid = 'scsi' . $1;
4292     }
4293     my $device = parse_drive($confid, $conf->{$confid});
4294     if ($device->{iothread}) {
4295         my $iothreads = vm_iothreads_list($vmid);
4296         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4297     }
4298 }
4299
4300 sub qemu_objectadd {
4301     my($vmid, $objectid, $qomtype) = @_;
4302
4303     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4304
4305     return 1;
4306 }
4307
4308 sub qemu_objectdel {
4309     my($vmid, $objectid) = @_;
4310
4311     vm_mon_cmd($vmid, "object-del", id => $objectid);
4312
4313     return 1;
4314 }
4315
4316 sub qemu_driveadd {
4317     my ($storecfg, $vmid, $device) = @_;
4318
4319     my $drive = print_drive_full($storecfg, $vmid, $device);
4320     $drive =~ s/\\/\\\\/g;
4321     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4322
4323     # If the command succeeds qemu prints: "OK"
4324     return 1 if $ret =~ m/OK/s;
4325
4326     die "adding drive failed: $ret\n";
4327 }
4328
4329 sub qemu_drivedel {
4330     my($vmid, $deviceid) = @_;
4331
4332     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4333     $ret =~ s/^\s+//;
4334
4335     return 1 if $ret eq "";
4336
4337     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4338     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4339
4340     die "deleting drive $deviceid failed : $ret\n";
4341 }
4342
4343 sub qemu_deviceaddverify {
4344     my ($vmid, $deviceid) = @_;
4345
4346     for (my $i = 0; $i <= 5; $i++) {
4347          my $devices_list = vm_devices_list($vmid);
4348          return 1 if defined($devices_list->{$deviceid});
4349          sleep 1;
4350     }
4351
4352     die "error on hotplug device '$deviceid'\n";
4353 }
4354
4355
4356 sub qemu_devicedelverify {
4357     my ($vmid, $deviceid) = @_;
4358
4359     # need to verify that the device is correctly removed as device_del
4360     # is async and empty return is not reliable
4361
4362     for (my $i = 0; $i <= 5; $i++) {
4363          my $devices_list = vm_devices_list($vmid);
4364          return 1 if !defined($devices_list->{$deviceid});
4365          sleep 1;
4366     }
4367
4368     die "error on hot-unplugging device '$deviceid'\n";
4369 }
4370
4371 sub qemu_findorcreatescsihw {
4372     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4373
4374     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4375
4376     my $scsihwid="$controller_prefix$controller";
4377     my $devices_list = vm_devices_list($vmid);
4378
4379     if(!defined($devices_list->{$scsihwid})) {
4380         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4381     }
4382
4383     return 1;
4384 }
4385
4386 sub qemu_deletescsihw {
4387     my ($conf, $vmid, $opt) = @_;
4388
4389     my $device = parse_drive($opt, $conf->{$opt});
4390
4391     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4392         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4393         return 1;
4394     }
4395
4396     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4397
4398     my $devices_list = vm_devices_list($vmid);
4399     foreach my $opt (keys %{$devices_list}) {
4400         if (PVE::QemuServer::is_valid_drivename($opt)) {
4401             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4402             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4403                 return 1;
4404             }
4405         }
4406     }
4407
4408     my $scsihwid="scsihw$controller";
4409
4410     vm_deviceunplug($vmid, $conf, $scsihwid);
4411
4412     return 1;
4413 }
4414
4415 sub qemu_add_pci_bridge {
4416     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4417
4418     my $bridges = {};
4419
4420     my $bridgeid;
4421
4422     print_pci_addr($device, $bridges, $arch, $machine_type);
4423
4424     while (my ($k, $v) = each %$bridges) {
4425         $bridgeid = $k;
4426     }
4427     return 1 if !defined($bridgeid) || $bridgeid < 1;
4428
4429     my $bridge = "pci.$bridgeid";
4430     my $devices_list = vm_devices_list($vmid);
4431
4432     if (!defined($devices_list->{$bridge})) {
4433         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4434     }
4435
4436     return 1;
4437 }
4438
4439 sub qemu_set_link_status {
4440     my ($vmid, $device, $up) = @_;
4441
4442     vm_mon_cmd($vmid, "set_link", name => $device,
4443                up => $up ? JSON::true : JSON::false);
4444 }
4445
4446 sub qemu_netdevadd {
4447     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4448
4449     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4450     my %options =  split(/[=,]/, $netdev);
4451
4452     vm_mon_cmd($vmid, "netdev_add",  %options);
4453     return 1;
4454 }
4455
4456 sub qemu_netdevdel {
4457     my ($vmid, $deviceid) = @_;
4458
4459     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4460 }
4461
4462 sub qemu_usb_hotplug {
4463     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4464
4465     return if !$device;
4466
4467     # remove the old one first
4468     vm_deviceunplug($vmid, $conf, $deviceid);
4469
4470     # check if xhci controller is necessary and available
4471     if ($device->{usb3}) {
4472
4473         my $devicelist = vm_devices_list($vmid);
4474
4475         if (!$devicelist->{xhci}) {
4476             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4477             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4478         }
4479     }
4480     my $d = parse_usb_device($device->{host});
4481     $d->{usb3} = $device->{usb3};
4482
4483     # add the new one
4484     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4485 }
4486
4487 sub qemu_cpu_hotplug {
4488     my ($vmid, $conf, $vcpus) = @_;
4489
4490     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4491
4492     my $sockets = 1;
4493     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4494     $sockets = $conf->{sockets} if  $conf->{sockets};
4495     my $cores = $conf->{cores} || 1;
4496     my $maxcpus = $sockets * $cores;
4497
4498     $vcpus = $maxcpus if !$vcpus;
4499
4500     die "you can't add more vcpus than maxcpus\n"
4501         if $vcpus > $maxcpus;
4502
4503     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4504
4505     if ($vcpus < $currentvcpus) {
4506
4507         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4508
4509             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4510                 qemu_devicedel($vmid, "cpu$i");
4511                 my $retry = 0;
4512                 my $currentrunningvcpus = undef;
4513                 while (1) {
4514                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4515                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4516                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4517                     $retry++;
4518                     sleep 1;
4519                 }
4520                 #update conf after each succesfull cpu unplug
4521                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4522                 PVE::QemuConfig->write_config($vmid, $conf);
4523             }
4524         } else {
4525             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4526         }
4527
4528         return;
4529     }
4530
4531     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4532     die "vcpus in running vm does not match its configuration\n"
4533         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4534
4535     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4536
4537         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4538             my $cpustr = print_cpu_device($conf, $i);
4539             qemu_deviceadd($vmid, $cpustr);
4540
4541             my $retry = 0;
4542             my $currentrunningvcpus = undef;
4543             while (1) {
4544                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4545                 last if scalar(@{$currentrunningvcpus}) == $i;
4546                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4547                 sleep 1;
4548                 $retry++;
4549             }
4550             #update conf after each succesfull cpu hotplug
4551             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4552             PVE::QemuConfig->write_config($vmid, $conf);
4553         }
4554     } else {
4555
4556         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4557             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4558         }
4559     }
4560 }
4561
4562 sub qemu_block_set_io_throttle {
4563     my ($vmid, $deviceid,
4564         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4565         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4566         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4567         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4568
4569     return if !check_running($vmid) ;
4570
4571     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4572         bps => int($bps),
4573         bps_rd => int($bps_rd),
4574         bps_wr => int($bps_wr),
4575         iops => int($iops),
4576         iops_rd => int($iops_rd),
4577         iops_wr => int($iops_wr),
4578         bps_max => int($bps_max),
4579         bps_rd_max => int($bps_rd_max),
4580         bps_wr_max => int($bps_wr_max),
4581         iops_max => int($iops_max),
4582         iops_rd_max => int($iops_rd_max),
4583         iops_wr_max => int($iops_wr_max),
4584         bps_max_length => int($bps_max_length),
4585         bps_rd_max_length => int($bps_rd_max_length),
4586         bps_wr_max_length => int($bps_wr_max_length),
4587         iops_max_length => int($iops_max_length),
4588         iops_rd_max_length => int($iops_rd_max_length),
4589         iops_wr_max_length => int($iops_wr_max_length),
4590     );
4591
4592 }
4593
4594 # old code, only used to shutdown old VM after update
4595 sub __read_avail {
4596     my ($fh, $timeout) = @_;
4597
4598     my $sel = new IO::Select;
4599     $sel->add($fh);
4600
4601     my $res = '';
4602     my $buf;
4603
4604     my @ready;
4605     while (scalar (@ready = $sel->can_read($timeout))) {
4606         my $count;
4607         if ($count = $fh->sysread($buf, 8192)) {
4608             if ($buf =~ /^(.*)\(qemu\) $/s) {
4609                 $res .= $1;
4610                 last;
4611             } else {
4612                 $res .= $buf;
4613             }
4614         } else {
4615             if (!defined($count)) {
4616                 die "$!\n";
4617             }
4618             last;
4619         }
4620     }
4621
4622     die "monitor read timeout\n" if !scalar(@ready);
4623
4624     return $res;
4625 }
4626
4627 sub qemu_block_resize {
4628     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4629
4630     my $running = check_running($vmid);
4631
4632     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4633
4634     return if !$running;
4635
4636     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4637
4638 }
4639
4640 sub qemu_volume_snapshot {
4641     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4642
4643     my $running = check_running($vmid);
4644
4645     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4646         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4647     } else {
4648         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4649     }
4650 }
4651
4652 sub qemu_volume_snapshot_delete {
4653     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4654
4655     my $running = check_running($vmid);
4656
4657     if($running) {
4658
4659         $running = undef;
4660         my $conf = PVE::QemuConfig->load_config($vmid);
4661         foreach_drive($conf, sub {
4662             my ($ds, $drive) = @_;
4663             $running = 1 if $drive->{file} eq $volid;
4664         });
4665     }
4666
4667     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4668         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4669     } else {
4670         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4671     }
4672 }
4673
4674 sub set_migration_caps {
4675     my ($vmid) = @_;
4676
4677     my $cap_ref = [];
4678
4679     my $enabled_cap = {
4680         "auto-converge" => 1,
4681         "xbzrle" => 1,
4682         "x-rdma-pin-all" => 0,
4683         "zero-blocks" => 0,
4684         "compress" => 0
4685     };
4686
4687     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4688
4689     for my $supported_capability (@$supported_capabilities) {
4690         push @$cap_ref, {
4691             capability => $supported_capability->{capability},
4692             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4693         };
4694     }
4695
4696     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4697 }
4698
4699 my $fast_plug_option = {
4700     'lock' => 1,
4701     'name' => 1,
4702     'onboot' => 1,
4703     'shares' => 1,
4704     'startup' => 1,
4705     'description' => 1,
4706     'protection' => 1,
4707     'vmstatestorage' => 1,
4708     'hookscript' => 1,
4709 };
4710
4711 # hotplug changes in [PENDING]
4712 # $selection hash can be used to only apply specified options, for
4713 # example: { cores => 1 } (only apply changed 'cores')
4714 # $errors ref is used to return error messages
4715 sub vmconfig_hotplug_pending {
4716     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4717
4718     my $defaults = load_defaults();
4719     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4720
4721     # commit values which do not have any impact on running VM first
4722     # Note: those option cannot raise errors, we we do not care about
4723     # $selection and always apply them.
4724
4725     my $add_error = sub {
4726         my ($opt, $msg) = @_;
4727         $errors->{$opt} = "hotplug problem - $msg";
4728     };
4729
4730     my $changes = 0;
4731     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4732         if ($fast_plug_option->{$opt}) {
4733             $conf->{$opt} = $conf->{pending}->{$opt};
4734             delete $conf->{pending}->{$opt};
4735             $changes = 1;
4736         }
4737     }
4738
4739     if ($changes) {
4740         PVE::QemuConfig->write_config($vmid, $conf);
4741         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4742     }
4743
4744     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4745
4746     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4747     while (my ($opt, $force) = each %$pending_delete_hash) {
4748         next if $selection && !$selection->{$opt};
4749         eval {
4750             if ($opt eq 'hotplug') {
4751                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4752             } elsif ($opt eq 'tablet') {
4753                 die "skip\n" if !$hotplug_features->{usb};
4754                 if ($defaults->{tablet}) {
4755                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4756                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4757                         if $arch eq 'aarch64';
4758                 } else {
4759                     vm_deviceunplug($vmid, $conf, 'tablet');
4760                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4761                 }
4762             } elsif ($opt =~ m/^usb\d+/) {
4763                 die "skip\n";
4764                 # since we cannot reliably hot unplug usb devices
4765                 # we are disabling it
4766                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4767                 vm_deviceunplug($vmid, $conf, $opt);
4768             } elsif ($opt eq 'vcpus') {
4769                 die "skip\n" if !$hotplug_features->{cpu};
4770                 qemu_cpu_hotplug($vmid, $conf, undef);
4771             } elsif ($opt eq 'balloon') {
4772                 # enable balloon device is not hotpluggable
4773                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4774                 # here we reset the ballooning value to memory
4775                 my $balloon = $conf->{memory} || $defaults->{memory};
4776                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4777             } elsif ($fast_plug_option->{$opt}) {
4778                 # do nothing
4779             } elsif ($opt =~ m/^net(\d+)$/) {
4780                 die "skip\n" if !$hotplug_features->{network};
4781                 vm_deviceunplug($vmid, $conf, $opt);
4782             } elsif (is_valid_drivename($opt)) {
4783                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4784                 vm_deviceunplug($vmid, $conf, $opt);
4785                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4786             } elsif ($opt =~ m/^memory$/) {
4787                 die "skip\n" if !$hotplug_features->{memory};
4788                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4789             } elsif ($opt eq 'cpuunits') {
4790                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4791             } elsif ($opt eq 'cpulimit') {
4792                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4793             } else {
4794                 die "skip\n";
4795             }
4796         };
4797         if (my $err = $@) {
4798             &$add_error($opt, $err) if $err ne "skip\n";
4799         } else {
4800             # save new config if hotplug was successful
4801             delete $conf->{$opt};
4802             vmconfig_undelete_pending_option($conf, $opt);
4803             PVE::QemuConfig->write_config($vmid, $conf);
4804             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4805         }
4806     }
4807
4808     my $apply_pending_cloudinit;
4809     $apply_pending_cloudinit = sub {
4810         my ($key, $value) = @_;
4811         $apply_pending_cloudinit = sub {}; # once is enough
4812
4813         my @cloudinit_opts = keys %$confdesc_cloudinit;
4814         foreach my $opt (keys %{$conf->{pending}}) {
4815             next if !grep { $_ eq $opt } @cloudinit_opts;
4816             $conf->{$opt} = delete $conf->{pending}->{$opt};
4817         }
4818
4819         my $new_conf = { %$conf };
4820         $new_conf->{$key} = $value;
4821         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4822     };
4823
4824     foreach my $opt (keys %{$conf->{pending}}) {
4825         next if $selection && !$selection->{$opt};
4826         my $value = $conf->{pending}->{$opt};
4827         eval {
4828             if ($opt eq 'hotplug') {
4829                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4830             } elsif ($opt eq 'tablet') {
4831                 die "skip\n" if !$hotplug_features->{usb};
4832                 if ($value == 1) {
4833                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4834                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4835                         if $arch eq 'aarch64';
4836                 } elsif ($value == 0) {
4837                     vm_deviceunplug($vmid, $conf, 'tablet');
4838                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4839                 }
4840             } elsif ($opt =~ m/^usb\d+$/) {
4841                 die "skip\n";
4842                 # since we cannot reliably hot unplug usb devices
4843                 # we are disabling it
4844                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4845                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4846                 die "skip\n" if !$d;
4847                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4848             } elsif ($opt eq 'vcpus') {
4849                 die "skip\n" if !$hotplug_features->{cpu};
4850                 qemu_cpu_hotplug($vmid, $conf, $value);
4851             } elsif ($opt eq 'balloon') {
4852                 # enable/disable balloning device is not hotpluggable
4853                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4854                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4855                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4856
4857                 # allow manual ballooning if shares is set to zero
4858                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4859                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4860                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4861                 }
4862             } elsif ($opt =~ m/^net(\d+)$/) {
4863                 # some changes can be done without hotplug
4864                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4865                                     $vmid, $opt, $value, $arch, $machine_type);
4866             } elsif (is_valid_drivename($opt)) {
4867                 # some changes can be done without hotplug
4868                 my $drive = parse_drive($opt, $value);
4869                 if (drive_is_cloudinit($drive)) {
4870                     &$apply_pending_cloudinit($opt, $value);
4871                 }
4872                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4873                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4874             } elsif ($opt =~ m/^memory$/) { #dimms
4875                 die "skip\n" if !$hotplug_features->{memory};
4876                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4877             } elsif ($opt eq 'cpuunits') {
4878                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4879             } elsif ($opt eq 'cpulimit') {
4880                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4881                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4882             } else {
4883                 die "skip\n";  # skip non-hot-pluggable options
4884             }
4885         };
4886         if (my $err = $@) {
4887             &$add_error($opt, $err) if $err ne "skip\n";
4888         } else {
4889             # save new config if hotplug was successful
4890             $conf->{$opt} = $value;
4891             delete $conf->{pending}->{$opt};
4892             PVE::QemuConfig->write_config($vmid, $conf);
4893             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4894         }
4895     }
4896 }
4897
4898 sub try_deallocate_drive {
4899     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4900
4901     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4902         my $volid = $drive->{file};
4903         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4904             my $sid = PVE::Storage::parse_volume_id($volid);
4905             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4906
4907             # check if the disk is really unused
4908             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4909                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4910             PVE::Storage::vdisk_free($storecfg, $volid);
4911             return 1;
4912         } else {
4913             # If vm is not owner of this disk remove from config
4914             return 1;
4915         }
4916     }
4917
4918     return undef;
4919 }
4920
4921 sub vmconfig_delete_or_detach_drive {
4922     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4923
4924     my $drive = parse_drive($opt, $conf->{$opt});
4925
4926     my $rpcenv = PVE::RPCEnvironment::get();
4927     my $authuser = $rpcenv->get_user();
4928
4929     if ($force) {
4930         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4931         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4932     } else {
4933         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4934     }
4935 }
4936
4937 sub vmconfig_apply_pending {
4938     my ($vmid, $conf, $storecfg) = @_;
4939
4940     # cold plug
4941
4942     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4943     while (my ($opt, $force) = each %$pending_delete_hash) {
4944         die "internal error" if $opt =~ m/^unused/;
4945         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4946         if (!defined($conf->{$opt})) {
4947             vmconfig_undelete_pending_option($conf, $opt);
4948             PVE::QemuConfig->write_config($vmid, $conf);
4949         } elsif (is_valid_drivename($opt)) {
4950             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4951             vmconfig_undelete_pending_option($conf, $opt);
4952             delete $conf->{$opt};
4953             PVE::QemuConfig->write_config($vmid, $conf);
4954         } else {
4955             vmconfig_undelete_pending_option($conf, $opt);
4956             delete $conf->{$opt};
4957             PVE::QemuConfig->write_config($vmid, $conf);
4958         }
4959     }
4960
4961     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4962
4963     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4964         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4965
4966         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4967             # skip if nothing changed
4968         } elsif (is_valid_drivename($opt)) {
4969             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4970                 if defined($conf->{$opt});
4971             $conf->{$opt} = $conf->{pending}->{$opt};
4972         } else {
4973             $conf->{$opt} = $conf->{pending}->{$opt};
4974         }
4975
4976         delete $conf->{pending}->{$opt};
4977         PVE::QemuConfig->write_config($vmid, $conf);
4978     }
4979 }
4980
4981 my $safe_num_ne = sub {
4982     my ($a, $b) = @_;
4983
4984     return 0 if !defined($a) && !defined($b);
4985     return 1 if !defined($a);
4986     return 1 if !defined($b);
4987
4988     return $a != $b;
4989 };
4990
4991 my $safe_string_ne = sub {
4992     my ($a, $b) = @_;
4993
4994     return 0 if !defined($a) && !defined($b);
4995     return 1 if !defined($a);
4996     return 1 if !defined($b);
4997
4998     return $a ne $b;
4999 };
5000
5001 sub vmconfig_update_net {
5002     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5003
5004     my $newnet = parse_net($value);
5005
5006     if ($conf->{$opt}) {
5007         my $oldnet = parse_net($conf->{$opt});
5008
5009         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5010             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5011             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5012             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5013
5014             # for non online change, we try to hot-unplug
5015             die "skip\n" if !$hotplug;
5016             vm_deviceunplug($vmid, $conf, $opt);
5017         } else {
5018
5019             die "internal error" if $opt !~ m/net(\d+)/;
5020             my $iface = "tap${vmid}i$1";
5021
5022             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5023                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5024                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5025                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5026                 PVE::Network::tap_unplug($iface);
5027                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5028             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5029                 # Rate can be applied on its own but any change above needs to
5030                 # include the rate in tap_plug since OVS resets everything.
5031                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5032             }
5033
5034             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5035                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5036             }
5037
5038             return 1;
5039         }
5040     }
5041
5042     if ($hotplug) {
5043         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5044     } else {
5045         die "skip\n";
5046     }
5047 }
5048
5049 sub vmconfig_update_disk {
5050     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5051
5052     # fixme: do we need force?
5053
5054     my $drive = parse_drive($opt, $value);
5055
5056     if ($conf->{$opt}) {
5057
5058         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5059
5060             my $media = $drive->{media} || 'disk';
5061             my $oldmedia = $old_drive->{media} || 'disk';
5062             die "unable to change media type\n" if $media ne $oldmedia;
5063
5064             if (!drive_is_cdrom($old_drive)) {
5065
5066                 if ($drive->{file} ne $old_drive->{file}) {
5067
5068                     die "skip\n" if !$hotplug;
5069
5070                     # unplug and register as unused
5071                     vm_deviceunplug($vmid, $conf, $opt);
5072                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5073
5074                 } else {
5075                     # update existing disk
5076
5077                     # skip non hotpluggable value
5078                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5079                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5080                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5081                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5082                         die "skip\n";
5083                     }
5084
5085                     # apply throttle
5086                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5087                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5088                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5089                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5090                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5091                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5092                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5093                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5094                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5095                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5096                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5097                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5098                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5099                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5100                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5101                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5102                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5103                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5104
5105                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5106                                                    ($drive->{mbps} || 0)*1024*1024,
5107                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5108                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5109                                                    $drive->{iops} || 0,
5110                                                    $drive->{iops_rd} || 0,
5111                                                    $drive->{iops_wr} || 0,
5112                                                    ($drive->{mbps_max} || 0)*1024*1024,
5113                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5114                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5115                                                    $drive->{iops_max} || 0,
5116                                                    $drive->{iops_rd_max} || 0,
5117                                                    $drive->{iops_wr_max} || 0,
5118                                                    $drive->{bps_max_length} || 1,
5119                                                    $drive->{bps_rd_max_length} || 1,
5120                                                    $drive->{bps_wr_max_length} || 1,
5121                                                    $drive->{iops_max_length} || 1,
5122                                                    $drive->{iops_rd_max_length} || 1,
5123                                                    $drive->{iops_wr_max_length} || 1);
5124
5125                     }
5126
5127                     return 1;
5128                 }
5129
5130             } else { # cdrom
5131
5132                 if ($drive->{file} eq 'none') {
5133                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5134                     if (drive_is_cloudinit($old_drive)) {
5135                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5136                     }
5137                 } else {
5138                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5139                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5140                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5141                 }
5142
5143                 return 1;
5144             }
5145         }
5146     }
5147
5148     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5149     # hotplug new disks
5150     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5151     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5152 }
5153
5154 sub vm_start {
5155     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5156         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5157
5158     PVE::QemuConfig->lock_config($vmid, sub {
5159         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5160
5161         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5162
5163         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5164
5165         PVE::QemuConfig->check_lock($conf)
5166             if !($skiplock || $is_suspended);
5167
5168         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5169
5170         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5171             vmconfig_apply_pending($vmid, $conf, $storecfg);
5172             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5173         }
5174
5175         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5176
5177         my $defaults = load_defaults();
5178
5179         # set environment variable useful inside network script
5180         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5181
5182         my $local_volumes = {};
5183
5184         if ($targetstorage) {
5185             foreach_drive($conf, sub {
5186                 my ($ds, $drive) = @_;
5187
5188                 return if drive_is_cdrom($drive);
5189
5190                 my $volid = $drive->{file};
5191
5192                 return if !$volid;
5193
5194                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5195
5196                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5197                 return if $scfg->{shared};
5198                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5199             });
5200
5201             my $format = undef;
5202
5203             foreach my $opt (sort keys %$local_volumes) {
5204
5205                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5206                 my $drive = parse_drive($opt, $conf->{$opt});
5207
5208                 #if remote storage is specified, use default format
5209                 if ($targetstorage && $targetstorage ne "1") {
5210                     $storeid = $targetstorage;
5211                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5212                     $format = $defFormat;
5213                 } else {
5214                     #else we use same format than original
5215                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5216                     $format = qemu_img_format($scfg, $volid);
5217                 }
5218
5219                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5220                 my $newdrive = $drive;
5221                 $newdrive->{format} = $format;
5222                 $newdrive->{file} = $newvolid;
5223                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5224                 $local_volumes->{$opt} = $drivestr;
5225                 #pass drive to conf for command line
5226                 $conf->{$opt} = $drivestr;
5227             }
5228         }
5229
5230         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5231
5232         if ($is_suspended) {
5233             # enforce machine type on suspended vm to ensure HW compatibility
5234             $forcemachine = $conf->{runningmachine};
5235             print "Resuming suspended VM\n";
5236         }
5237
5238         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5239
5240         my $migrate_port = 0;
5241         my $migrate_uri;
5242         if ($statefile) {
5243             if ($statefile eq 'tcp') {
5244                 my $localip = "localhost";
5245                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5246                 my $nodename = PVE::INotify::nodename();
5247
5248                 if (!defined($migration_type)) {
5249                     if (defined($datacenterconf->{migration}->{type})) {
5250                         $migration_type = $datacenterconf->{migration}->{type};
5251                     } else {
5252                         $migration_type = 'secure';
5253                     }
5254                 }
5255
5256                 if ($migration_type eq 'insecure') {
5257                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5258                     if ($migrate_network_addr) {
5259                         $localip = $migrate_network_addr;
5260                     } else {
5261                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5262                     }
5263
5264                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5265                 }
5266
5267                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5268                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5269                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5270                 push @$cmd, '-incoming', $migrate_uri;
5271                 push @$cmd, '-S';
5272
5273             } elsif ($statefile eq 'unix') {
5274                 # should be default for secure migrations as a ssh TCP forward
5275                 # tunnel is not deterministic reliable ready and fails regurarly
5276                 # to set up in time, so use UNIX socket forwards
5277                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5278                 unlink $socket_addr;
5279
5280                 $migrate_uri = "unix:$socket_addr";
5281
5282                 push @$cmd, '-incoming', $migrate_uri;
5283                 push @$cmd, '-S';
5284
5285             } else {
5286                 push @$cmd, '-loadstate', $statefile;
5287             }
5288         } elsif ($paused) {
5289             push @$cmd, '-S';
5290         }
5291
5292         # host pci devices
5293         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5294           my $d = parse_hostpci($conf->{"hostpci$i"});
5295           next if !$d;
5296           my $pcidevices = $d->{pciid};
5297           foreach my $pcidevice (@$pcidevices) {
5298                 my $pciid = $pcidevice->{id};
5299
5300                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5301                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5302                 die "no pci device info for device '$pciid'\n" if !$info;
5303
5304                 if ($d->{mdev}) {
5305                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5306                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5307                 } else {
5308                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5309                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5310                     die "can't reset pci device '$pciid'\n"
5311                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5312                 }
5313           }
5314         }
5315
5316         PVE::Storage::activate_volumes($storecfg, $vollist);
5317
5318         if (-d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
5319             eval {
5320                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5321                     outfunc => sub {}, errfunc => sub {});
5322             };
5323         }
5324
5325         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5326                                                   : $defaults->{cpuunits};
5327
5328         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5329         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5330
5331         my %properties = (
5332             Slice => 'qemu.slice',
5333             KillMode => 'none',
5334             CPUShares => $cpuunits
5335         );
5336
5337         if (my $cpulimit = $conf->{cpulimit}) {
5338             $properties{CPUQuota} = int($cpulimit * 100);
5339         }
5340         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5341
5342         my $run_qemu = sub {
5343             PVE::Tools::run_fork sub {
5344                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5345                 run_command($cmd, %run_params);
5346             };
5347         };
5348
5349         if ($conf->{hugepages}) {
5350
5351             my $code = sub {
5352                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5353                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5354
5355                 PVE::QemuServer::Memory::hugepages_mount();
5356                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5357
5358                 eval { $run_qemu->() };
5359                 if (my $err = $@) {
5360                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5361                     die $err;
5362                 }
5363
5364                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5365             };
5366             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5367
5368         } else {
5369             eval { $run_qemu->() };
5370         }
5371
5372         if (my $err = $@) {
5373             # deactivate volumes if start fails
5374             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5375             die "start failed: $err";
5376         }
5377
5378         print "migration listens on $migrate_uri\n" if $migrate_uri;
5379
5380         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5381             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5382             warn $@ if $@;
5383         }
5384
5385         #start nbd server for storage migration
5386         if ($targetstorage) {
5387             my $nodename = PVE::INotify::nodename();
5388             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5389             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5390             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5391             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5392
5393             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5394
5395             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5396
5397             foreach my $opt (sort keys %$local_volumes) {
5398                 my $volid = $local_volumes->{$opt};
5399                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5400                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5401                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5402             }
5403         }
5404
5405         if ($migratedfrom) {
5406             eval {
5407                 set_migration_caps($vmid);
5408             };
5409             warn $@ if $@;
5410
5411             if ($spice_port) {
5412                 print "spice listens on port $spice_port\n";
5413                 if ($spice_ticket) {
5414                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5415                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5416                 }
5417             }
5418
5419         } else {
5420             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5421                 if !$statefile && $conf->{balloon};
5422
5423             foreach my $opt (keys %$conf) {
5424                 next if $opt !~  m/^net\d+$/;
5425                 my $nicconf = parse_net($conf->{$opt});
5426                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5427             }
5428         }
5429
5430         vm_mon_cmd_nocheck($vmid, 'qom-set',
5431                     path => "machine/peripheral/balloon0",
5432                     property => "guest-stats-polling-interval",
5433                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5434
5435         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5436             print "Resumed VM, removing state\n";
5437             delete $conf->@{qw(lock vmstate runningmachine)};
5438             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5439             PVE::Storage::vdisk_free($storecfg, $vmstate);
5440             PVE::QemuConfig->write_config($vmid, $conf);
5441         }
5442
5443         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5444     });
5445 }
5446
5447 sub vm_mon_cmd {
5448     my ($vmid, $execute, %params) = @_;
5449
5450     my $cmd = { execute => $execute, arguments => \%params };
5451     vm_qmp_command($vmid, $cmd);
5452 }
5453
5454 sub vm_mon_cmd_nocheck {
5455     my ($vmid, $execute, %params) = @_;
5456
5457     my $cmd = { execute => $execute, arguments => \%params };
5458     vm_qmp_command($vmid, $cmd, 1);
5459 }
5460
5461 sub vm_qmp_command {
5462     my ($vmid, $cmd, $nocheck) = @_;
5463
5464     my $res;
5465
5466     my $timeout;
5467     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5468         $timeout = $cmd->{arguments}->{timeout};
5469         delete $cmd->{arguments}->{timeout};
5470     }
5471
5472     eval {
5473         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5474         my $sname = qmp_socket($vmid);
5475         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5476             my $qmpclient = PVE::QMPClient->new();
5477
5478             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5479         } else {
5480             die "unable to open monitor socket\n";
5481         }
5482     };
5483     if (my $err = $@) {
5484         syslog("err", "VM $vmid qmp command failed - $err");
5485         die $err;
5486     }
5487
5488     return $res;
5489 }
5490
5491 sub vm_human_monitor_command {
5492     my ($vmid, $cmdline) = @_;
5493
5494     my $res;
5495
5496     my $cmd = {
5497         execute => 'human-monitor-command',
5498         arguments => { 'command-line' => $cmdline},
5499     };
5500
5501     return vm_qmp_command($vmid, $cmd);
5502 }
5503
5504 sub vm_commandline {
5505     my ($storecfg, $vmid, $snapname) = @_;
5506
5507     my $conf = PVE::QemuConfig->load_config($vmid);
5508
5509     if ($snapname) {
5510         my $snapshot = $conf->{snapshots}->{$snapname};
5511         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5512
5513         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5514
5515         $conf = $snapshot;
5516     }
5517
5518     my $defaults = load_defaults();
5519
5520     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5521
5522     return PVE::Tools::cmd2string($cmd);
5523 }
5524
5525 sub vm_reset {
5526     my ($vmid, $skiplock) = @_;
5527
5528     PVE::QemuConfig->lock_config($vmid, sub {
5529
5530         my $conf = PVE::QemuConfig->load_config($vmid);
5531
5532         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5533
5534         vm_mon_cmd($vmid, "system_reset");
5535     });
5536 }
5537
5538 sub get_vm_volumes {
5539     my ($conf) = @_;
5540
5541     my $vollist = [];
5542     foreach_volid($conf, sub {
5543         my ($volid, $attr) = @_;
5544
5545         return if $volid =~ m|^/|;
5546
5547         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5548         return if !$sid;
5549
5550         push @$vollist, $volid;
5551     });
5552
5553     return $vollist;
5554 }
5555
5556 sub vm_stop_cleanup {
5557     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5558
5559     eval {
5560
5561         if (!$keepActive) {
5562             my $vollist = get_vm_volumes($conf);
5563             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5564         }
5565
5566         foreach my $ext (qw(mon qmp pid vnc qga)) {
5567             unlink "/var/run/qemu-server/${vmid}.$ext";
5568         }
5569
5570         if ($conf->{ivshmem}) {
5571             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5572             # just delete it for now, VMs which have this already open do not
5573             # are affected, but new VMs will get a separated one. If this
5574             # becomes an issue we either add some sort of ref-counting or just
5575             # add a "don't delete on stop" flag to the ivshmem format.
5576             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5577         }
5578
5579         foreach my $key (keys %$conf) {
5580             next if $key !~ m/^hostpci(\d+)$/;
5581             my $hostpciindex = $1;
5582             my $d = parse_hostpci($conf->{$key});
5583             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5584
5585             foreach my $pci (@{$d->{pciid}}) {
5586                 my $pciid = $pci->{id};
5587                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5588             }
5589         }
5590
5591         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5592     };
5593     warn $@ if $@; # avoid errors - just warn
5594 }
5595
5596 # Note: use $nockeck to skip tests if VM configuration file exists.
5597 # We need that when migration VMs to other nodes (files already moved)
5598 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5599 sub vm_stop {
5600     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5601
5602     $force = 1 if !defined($force) && !$shutdown;
5603
5604     if ($migratedfrom){
5605         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5606         kill 15, $pid if $pid;
5607         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5608         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5609         return;
5610     }
5611
5612     PVE::QemuConfig->lock_config($vmid, sub {
5613
5614         my $pid = check_running($vmid, $nocheck);
5615         return if !$pid;
5616
5617         my $conf;
5618         if (!$nocheck) {
5619             $conf = PVE::QemuConfig->load_config($vmid);
5620             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5621             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5622                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5623                 $timeout = $opts->{down} if $opts->{down};
5624             }
5625             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5626         }
5627
5628         $timeout = 60 if !defined($timeout);
5629
5630         eval {
5631             if ($shutdown) {
5632                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5633                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5634                 } else {
5635                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5636                 }
5637             } else {
5638                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5639             }
5640         };
5641         my $err = $@;
5642
5643         if (!$err) {
5644             my $count = 0;
5645             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5646                 $count++;
5647                 sleep 1;
5648             }
5649
5650             if ($count >= $timeout) {
5651                 if ($force) {
5652                     warn "VM still running - terminating now with SIGTERM\n";
5653                     kill 15, $pid;
5654                 } else {
5655                     die "VM quit/powerdown failed - got timeout\n";
5656                 }
5657             } else {
5658                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5659                 return;
5660             }
5661         } else {
5662             if ($force) {
5663                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5664                 kill 15, $pid;
5665             } else {
5666                 die "VM quit/powerdown failed\n";
5667             }
5668         }
5669
5670         # wait again
5671         $timeout = 10;
5672
5673         my $count = 0;
5674         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5675             $count++;
5676             sleep 1;
5677         }
5678
5679         if ($count >= $timeout) {
5680             warn "VM still running - terminating now with SIGKILL\n";
5681             kill 9, $pid;
5682             sleep 1;
5683         }
5684
5685         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5686    });
5687 }
5688
5689 sub vm_suspend {
5690     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5691
5692     my $conf;
5693     my $path;
5694     my $storecfg;
5695     my $vmstate;
5696
5697     PVE::QemuConfig->lock_config($vmid, sub {
5698
5699         $conf = PVE::QemuConfig->load_config($vmid);
5700
5701         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5702         PVE::QemuConfig->check_lock($conf)
5703             if !($skiplock || $is_backing_up);
5704
5705         die "cannot suspend to disk during backup\n"
5706             if $is_backing_up && $includestate;
5707
5708         if ($includestate) {
5709             $conf->{lock} = 'suspending';
5710             my $date = strftime("%Y-%m-%d", localtime(time()));
5711             $storecfg = PVE::Storage::config();
5712             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5713             $path = PVE::Storage::path($storecfg, $vmstate);
5714             PVE::QemuConfig->write_config($vmid, $conf);
5715         } else {
5716             vm_mon_cmd($vmid, "stop");
5717         }
5718     });
5719
5720     if ($includestate) {
5721         # save vm state
5722         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5723
5724         eval {
5725             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5726             for(;;) {
5727                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5728                 if (!$state->{status}) {
5729                     die "savevm not active\n";
5730                 } elsif ($state->{status} eq 'active') {
5731                     sleep(1);
5732                     next;
5733                 } elsif ($state->{status} eq 'completed') {
5734                     print "State saved, quitting\n";
5735                     last;
5736                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5737                     die "query-savevm failed with error '$state->{error}'\n"
5738                 } else {
5739                     die "query-savevm returned status '$state->{status}'\n";
5740                 }
5741             }
5742         };
5743         my $err = $@;
5744
5745         PVE::QemuConfig->lock_config($vmid, sub {
5746             $conf = PVE::QemuConfig->load_config($vmid);
5747             if ($err) {
5748                 # cleanup, but leave suspending lock, to indicate something went wrong
5749                 eval {
5750                     vm_mon_cmd($vmid, "savevm-end");
5751                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5752                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5753                     delete $conf->@{qw(vmstate runningmachine)};
5754                     PVE::QemuConfig->write_config($vmid, $conf);
5755                 };
5756                 warn $@ if $@;
5757                 die $err;
5758             }
5759
5760             die "lock changed unexpectedly\n"
5761                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5762
5763             vm_qmp_command($vmid, { execute => "quit" });
5764             $conf->{lock} = 'suspended';
5765             PVE::QemuConfig->write_config($vmid, $conf);
5766         });
5767     }
5768 }
5769
5770 sub vm_resume {
5771     my ($vmid, $skiplock, $nocheck) = @_;
5772
5773     PVE::QemuConfig->lock_config($vmid, sub {
5774
5775         my $res = vm_mon_cmd($vmid, 'query-status');
5776         my $resume_cmd = 'cont';
5777
5778         if ($res->{status} && $res->{status} eq 'suspended') {
5779             $resume_cmd = 'system_wakeup';
5780         }
5781
5782         if (!$nocheck) {
5783
5784             my $conf = PVE::QemuConfig->load_config($vmid);
5785
5786             PVE::QemuConfig->check_lock($conf)
5787                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5788
5789             vm_mon_cmd($vmid, $resume_cmd);
5790
5791         } else {
5792             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5793         }
5794     });
5795 }
5796
5797 sub vm_sendkey {
5798     my ($vmid, $skiplock, $key) = @_;
5799
5800     PVE::QemuConfig->lock_config($vmid, sub {
5801
5802         my $conf = PVE::QemuConfig->load_config($vmid);
5803
5804         # there is no qmp command, so we use the human monitor command
5805         vm_human_monitor_command($vmid, "sendkey $key");
5806     });
5807 }
5808
5809 sub vm_destroy {
5810     my ($storecfg, $vmid, $skiplock) = @_;
5811
5812     PVE::QemuConfig->lock_config($vmid, sub {
5813
5814         my $conf = PVE::QemuConfig->load_config($vmid);
5815
5816         if (!check_running($vmid)) {
5817             destroy_vm($storecfg, $vmid, undef, $skiplock);
5818         } else {
5819             die "VM $vmid is running - destroy failed\n";
5820         }
5821     });
5822 }
5823
5824 # vzdump restore implementaion
5825
5826 sub tar_archive_read_firstfile {
5827     my $archive = shift;
5828
5829     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5830
5831     # try to detect archive type first
5832     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5833         die "unable to open file '$archive'\n";
5834     my $firstfile = <$fh>;
5835     kill 15, $pid;
5836     close $fh;
5837
5838     die "ERROR: archive contaions no data\n" if !$firstfile;
5839     chomp $firstfile;
5840
5841     return $firstfile;
5842 }
5843
5844 sub tar_restore_cleanup {
5845     my ($storecfg, $statfile) = @_;
5846
5847     print STDERR "starting cleanup\n";
5848
5849     if (my $fd = IO::File->new($statfile, "r")) {
5850         while (defined(my $line = <$fd>)) {
5851             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5852                 my $volid = $2;
5853                 eval {
5854                     if ($volid =~ m|^/|) {
5855                         unlink $volid || die 'unlink failed\n';
5856                     } else {
5857                         PVE::Storage::vdisk_free($storecfg, $volid);
5858                     }
5859                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5860                 };
5861                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5862             } else {
5863                 print STDERR "unable to parse line in statfile - $line";
5864             }
5865         }
5866         $fd->close();
5867     }
5868 }
5869
5870 sub restore_archive {
5871     my ($archive, $vmid, $user, $opts) = @_;
5872
5873     my $format = $opts->{format};
5874     my $comp;
5875
5876     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5877         $format = 'tar' if !$format;
5878         $comp = 'gzip';
5879     } elsif ($archive =~ m/\.tar$/) {
5880         $format = 'tar' if !$format;
5881     } elsif ($archive =~ m/.tar.lzo$/) {
5882         $format = 'tar' if !$format;
5883         $comp = 'lzop';
5884     } elsif ($archive =~ m/\.vma$/) {
5885         $format = 'vma' if !$format;
5886     } elsif ($archive =~ m/\.vma\.gz$/) {
5887         $format = 'vma' if !$format;
5888         $comp = 'gzip';
5889     } elsif ($archive =~ m/\.vma\.lzo$/) {
5890         $format = 'vma' if !$format;
5891         $comp = 'lzop';
5892     } else {
5893         $format = 'vma' if !$format; # default
5894     }
5895
5896     # try to detect archive format
5897     if ($format eq 'tar') {
5898         return restore_tar_archive($archive, $vmid, $user, $opts);
5899     } else {
5900         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5901     }
5902 }
5903
5904 sub restore_update_config_line {
5905     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5906
5907     return if $line =~ m/^\#qmdump\#/;
5908     return if $line =~ m/^\#vzdump\#/;
5909     return if $line =~ m/^lock:/;
5910     return if $line =~ m/^unused\d+:/;
5911     return if $line =~ m/^parent:/;
5912     return if $line =~ m/^template:/; # restored VM is never a template
5913
5914     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5915     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5916         # try to convert old 1.X settings
5917         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5918         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5919             my ($model, $macaddr) = split(/\=/, $devconfig);
5920             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5921             my $net = {
5922                 model => $model,
5923                 bridge => "vmbr$ind",
5924                 macaddr => $macaddr,
5925             };
5926             my $netstr = print_net($net);
5927
5928             print $outfd "net$cookie->{netcount}: $netstr\n";
5929             $cookie->{netcount}++;
5930         }
5931     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5932         my ($id, $netstr) = ($1, $2);
5933         my $net = parse_net($netstr);
5934         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5935         $netstr = print_net($net);
5936         print $outfd "$id: $netstr\n";
5937     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5938         my $virtdev = $1;
5939         my $value = $3;
5940         my $di = parse_drive($virtdev, $value);
5941         if (defined($di->{backup}) && !$di->{backup}) {
5942             print $outfd "#$line";
5943         } elsif ($map->{$virtdev}) {
5944             delete $di->{format}; # format can change on restore
5945             $di->{file} = $map->{$virtdev};
5946             $value = print_drive($vmid, $di);
5947             print $outfd "$virtdev: $value\n";
5948         } else {
5949             print $outfd $line;
5950         }
5951     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5952         my $vmgenid = $1;
5953         if ($vmgenid ne '0') {
5954             # always generate a new vmgenid if there was a valid one setup
5955             $vmgenid = generate_uuid();
5956         }
5957         print $outfd "vmgenid: $vmgenid\n";
5958     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5959         my ($uuid, $uuid_str);
5960         UUID::generate($uuid);
5961         UUID::unparse($uuid, $uuid_str);
5962         my $smbios1 = parse_smbios1($2);
5963         $smbios1->{uuid} = $uuid_str;
5964         print $outfd $1.print_smbios1($smbios1)."\n";
5965     } else {
5966         print $outfd $line;
5967     }
5968 }
5969
5970 sub scan_volids {
5971     my ($cfg, $vmid) = @_;
5972
5973     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5974
5975     my $volid_hash = {};
5976     foreach my $storeid (keys %$info) {
5977         foreach my $item (@{$info->{$storeid}}) {
5978             next if !($item->{volid} && $item->{size});
5979             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5980             $volid_hash->{$item->{volid}} = $item;
5981         }
5982     }
5983
5984     return $volid_hash;
5985 }
5986
5987 sub is_volume_in_use {
5988     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5989
5990     my $path = PVE::Storage::path($storecfg, $volid);
5991
5992     my $scan_config = sub {
5993         my ($cref, $snapname) = @_;
5994
5995         foreach my $key (keys %$cref) {
5996             my $value = $cref->{$key};
5997             if (is_valid_drivename($key)) {
5998                 next if $skip_drive && $key eq $skip_drive;
5999                 my $drive = parse_drive($key, $value);
6000                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6001                 return 1 if $volid eq $drive->{file};
6002                 if ($drive->{file} =~ m!^/!) {
6003                     return 1 if $drive->{file} eq $path;
6004                 } else {
6005                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6006                     next if !$storeid;
6007                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6008                     next if !$scfg;
6009                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6010                 }
6011             }
6012         }
6013
6014         return 0;
6015     };
6016
6017     return 1 if &$scan_config($conf);
6018
6019     undef $skip_drive;
6020
6021     foreach my $snapname (keys %{$conf->{snapshots}}) {
6022         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6023     }
6024
6025     return 0;
6026 }
6027
6028 sub update_disksize {
6029     my ($vmid, $conf, $volid_hash) = @_;
6030
6031     my $changes;
6032     my $prefix = "VM $vmid:";
6033
6034     # used and unused disks
6035     my $referenced = {};
6036
6037     # Note: it is allowed to define multiple storages with same path (alias), so
6038     # we need to check both 'volid' and real 'path' (two different volid can point
6039     # to the same path).
6040
6041     my $referencedpath = {};
6042
6043     # update size info
6044     foreach my $opt (keys %$conf) {
6045         if (is_valid_drivename($opt)) {
6046             my $drive = parse_drive($opt, $conf->{$opt});
6047             my $volid = $drive->{file};
6048             next if !$volid;
6049
6050             $referenced->{$volid} = 1;
6051             if ($volid_hash->{$volid} &&
6052                 (my $path = $volid_hash->{$volid}->{path})) {
6053                 $referencedpath->{$path} = 1;
6054             }
6055
6056             next if drive_is_cdrom($drive);
6057             next if !$volid_hash->{$volid};
6058
6059             $drive->{size} = $volid_hash->{$volid}->{size};
6060             my $new = print_drive($vmid, $drive);
6061             if ($new ne $conf->{$opt}) {
6062                 $changes = 1;
6063                 $conf->{$opt} = $new;
6064                 print "$prefix update disk '$opt' information.\n";
6065             }
6066         }
6067     }
6068
6069     # remove 'unusedX' entry if volume is used
6070     foreach my $opt (keys %$conf) {
6071         next if $opt !~ m/^unused\d+$/;
6072         my $volid = $conf->{$opt};
6073         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6074         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6075             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6076             $changes = 1;
6077             delete $conf->{$opt};
6078         }
6079
6080         $referenced->{$volid} = 1;
6081         $referencedpath->{$path} = 1 if $path;
6082     }
6083
6084     foreach my $volid (sort keys %$volid_hash) {
6085         next if $volid =~ m/vm-$vmid-state-/;
6086         next if $referenced->{$volid};
6087         my $path = $volid_hash->{$volid}->{path};
6088         next if !$path; # just to be sure
6089         next if $referencedpath->{$path};
6090         $changes = 1;
6091         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6092         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6093         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6094     }
6095
6096     return $changes;
6097 }
6098
6099 sub rescan {
6100     my ($vmid, $nolock, $dryrun) = @_;
6101
6102     my $cfg = PVE::Storage::config();
6103
6104     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6105     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6106     foreach my $stor (keys %{$cfg->{ids}}) {
6107         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6108     }
6109
6110     print "rescan volumes...\n";
6111     my $volid_hash = scan_volids($cfg, $vmid);
6112
6113     my $updatefn =  sub {
6114         my ($vmid) = @_;
6115
6116         my $conf = PVE::QemuConfig->load_config($vmid);
6117
6118         PVE::QemuConfig->check_lock($conf);
6119
6120         my $vm_volids = {};
6121         foreach my $volid (keys %$volid_hash) {
6122             my $info = $volid_hash->{$volid};
6123             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6124         }
6125
6126         my $changes = update_disksize($vmid, $conf, $vm_volids);
6127
6128         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6129     };
6130
6131     if (defined($vmid)) {
6132         if ($nolock) {
6133             &$updatefn($vmid);
6134         } else {
6135             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6136         }
6137     } else {
6138         my $vmlist = config_list();
6139         foreach my $vmid (keys %$vmlist) {
6140             if ($nolock) {
6141                 &$updatefn($vmid);
6142             } else {
6143                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6144             }
6145         }
6146     }
6147 }
6148
6149 sub restore_vma_archive {
6150     my ($archive, $vmid, $user, $opts, $comp) = @_;
6151
6152     my $readfrom = $archive;
6153
6154     my $cfg = PVE::Storage::config();
6155     my $commands = [];
6156     my $bwlimit = $opts->{bwlimit};
6157
6158     my $dbg_cmdstring = '';
6159     my $add_pipe = sub {
6160         my ($cmd) = @_;
6161         push @$commands, $cmd;
6162         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6163         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6164         $readfrom = '-';
6165     };
6166
6167     my $input = undef;
6168     if ($archive eq '-') {
6169         $input = '<&STDIN';
6170     } else {
6171         # If we use a backup from a PVE defined storage we also consider that
6172         # storage's rate limit:
6173         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6174         if (defined($volid)) {
6175             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6176             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6177             if ($readlimit) {
6178                 print STDERR "applying read rate limit: $readlimit\n";
6179                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6180                 $add_pipe->($cstream);
6181             }
6182         }
6183     }
6184
6185     if ($comp) {
6186         my $cmd;
6187         if ($comp eq 'gzip') {
6188             $cmd = ['zcat', $readfrom];
6189         } elsif ($comp eq 'lzop') {
6190             $cmd = ['lzop', '-d', '-c', $readfrom];
6191         } else {
6192             die "unknown compression method '$comp'\n";
6193         }
6194         $add_pipe->($cmd);
6195     }
6196
6197     my $tmpdir = "/var/tmp/vzdumptmp$$";
6198     rmtree $tmpdir;
6199
6200     # disable interrupts (always do cleanups)
6201     local $SIG{INT} =
6202         local $SIG{TERM} =
6203         local $SIG{QUIT} =
6204         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6205
6206     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6207     POSIX::mkfifo($mapfifo, 0600);
6208     my $fifofh;
6209
6210     my $openfifo = sub {
6211         open($fifofh, '>', $mapfifo) || die $!;
6212     };
6213
6214     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6215
6216     my $oldtimeout;
6217     my $timeout = 5;
6218
6219     my $devinfo = {};
6220
6221     my $rpcenv = PVE::RPCEnvironment::get();
6222
6223     my $conffile = PVE::QemuConfig->config_file($vmid);
6224     my $tmpfn = "$conffile.$$.tmp";
6225
6226     # Note: $oldconf is undef if VM does not exists
6227     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6228     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6229
6230     my %storage_limits;
6231
6232     my $print_devmap = sub {
6233         my $virtdev_hash = {};
6234
6235         my $cfgfn = "$tmpdir/qemu-server.conf";
6236
6237         # we can read the config - that is already extracted
6238         my $fh = IO::File->new($cfgfn, "r") ||
6239             "unable to read qemu-server.conf - $!\n";
6240
6241         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6242         if (-f $fwcfgfn) {
6243             my $pve_firewall_dir = '/etc/pve/firewall';
6244             mkdir $pve_firewall_dir; # make sure the dir exists
6245             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6246         }
6247
6248         while (defined(my $line = <$fh>)) {
6249             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6250                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6251                 die "archive does not contain data for drive '$virtdev'\n"
6252                     if !$devinfo->{$devname};
6253                 if (defined($opts->{storage})) {
6254                     $storeid = $opts->{storage} || 'local';
6255                 } elsif (!$storeid) {
6256                     $storeid = 'local';
6257                 }
6258                 $format = 'raw' if !$format;
6259                 $devinfo->{$devname}->{devname} = $devname;
6260                 $devinfo->{$devname}->{virtdev} = $virtdev;
6261                 $devinfo->{$devname}->{format} = $format;
6262                 $devinfo->{$devname}->{storeid} = $storeid;
6263
6264                 # check permission on storage
6265                 my $pool = $opts->{pool}; # todo: do we need that?
6266                 if ($user ne 'root@pam') {
6267                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6268                 }
6269
6270                 $storage_limits{$storeid} = $bwlimit;
6271
6272                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6273             }
6274         }
6275
6276         foreach my $key (keys %storage_limits) {
6277             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6278             next if !$limit;
6279             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6280             $storage_limits{$key} = $limit * 1024;
6281         }
6282
6283         foreach my $devname (keys %$devinfo) {
6284             die "found no device mapping information for device '$devname'\n"
6285                 if !$devinfo->{$devname}->{virtdev};
6286         }
6287
6288         # create empty/temp config
6289         if ($oldconf) {
6290             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6291             foreach_drive($oldconf, sub {
6292                 my ($ds, $drive) = @_;
6293
6294                 return if drive_is_cdrom($drive);
6295
6296                 my $volid = $drive->{file};
6297
6298                 return if !$volid || $volid =~ m|^/|;
6299
6300                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6301                 return if !$path || !$owner || ($owner != $vmid);
6302
6303                 # Note: only delete disk we want to restore
6304                 # other volumes will become unused
6305                 if ($virtdev_hash->{$ds}) {
6306                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6307                     if (my $err = $@) {
6308                         warn $err;
6309                     }
6310                 }
6311             });
6312
6313             # delete vmstate files
6314             # since after the restore we have no snapshots anymore
6315             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6316                 my $snap = $oldconf->{snapshots}->{$snapname};
6317                 if ($snap->{vmstate}) {
6318                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6319                     if (my $err = $@) {
6320                         warn $err;
6321                     }
6322                 }
6323             }
6324         }
6325
6326         my $map = {};
6327         foreach my $virtdev (sort keys %$virtdev_hash) {
6328             my $d = $virtdev_hash->{$virtdev};
6329             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6330             my $storeid = $d->{storeid};
6331             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6332
6333             my $map_opts = '';
6334             if (my $limit = $storage_limits{$storeid}) {
6335                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6336             }
6337
6338             # test if requested format is supported
6339             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6340             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6341             $d->{format} = $defFormat if !$supported;
6342
6343             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6344                                                   $d->{format}, undef, $alloc_size);
6345             print STDERR "new volume ID is '$volid'\n";
6346             $d->{volid} = $volid;
6347             my $path = PVE::Storage::path($cfg, $volid);
6348
6349             PVE::Storage::activate_volumes($cfg,[$volid]);
6350
6351             my $write_zeros = 1;
6352             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6353                 $write_zeros = 0;
6354             }
6355
6356             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6357
6358             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6359             $map->{$virtdev} = $volid;
6360         }
6361
6362         $fh->seek(0, 0) || die "seek failed - $!\n";
6363
6364         my $outfd = new IO::File ($tmpfn, "w") ||
6365             die "unable to write config for VM $vmid\n";
6366
6367         my $cookie = { netcount => 0 };
6368         while (defined(my $line = <$fh>)) {
6369             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6370         }
6371
6372         $fh->close();
6373         $outfd->close();
6374     };
6375
6376     eval {
6377         # enable interrupts
6378         local $SIG{INT} =
6379             local $SIG{TERM} =
6380             local $SIG{QUIT} =
6381             local $SIG{HUP} =
6382             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6383         local $SIG{ALRM} = sub { die "got timeout\n"; };
6384
6385         $oldtimeout = alarm($timeout);
6386
6387         my $parser = sub {
6388             my $line = shift;
6389
6390             print "$line\n";
6391
6392             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6393                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6394                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6395             } elsif ($line =~ m/^CTIME: /) {
6396                 # we correctly received the vma config, so we can disable
6397                 # the timeout now for disk allocation (set to 10 minutes, so
6398                 # that we always timeout if something goes wrong)
6399                 alarm(600);
6400                 &$print_devmap();
6401                 print $fifofh "done\n";
6402                 my $tmp = $oldtimeout || 0;
6403                 $oldtimeout = undef;
6404                 alarm($tmp);
6405                 close($fifofh);
6406             }
6407         };
6408
6409         print "restore vma archive: $dbg_cmdstring\n";
6410         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6411     };
6412     my $err = $@;
6413
6414     alarm($oldtimeout) if $oldtimeout;
6415
6416     my $vollist = [];
6417     foreach my $devname (keys %$devinfo) {
6418         my $volid = $devinfo->{$devname}->{volid};
6419         push @$vollist, $volid if $volid;
6420     }
6421
6422     PVE::Storage::deactivate_volumes($cfg, $vollist);
6423
6424     unlink $mapfifo;
6425
6426     if ($err) {
6427         rmtree $tmpdir;
6428         unlink $tmpfn;
6429
6430         foreach my $devname (keys %$devinfo) {
6431             my $volid = $devinfo->{$devname}->{volid};
6432             next if !$volid;
6433             eval {
6434                 if ($volid =~ m|^/|) {
6435                     unlink $volid || die 'unlink failed\n';
6436                 } else {
6437                     PVE::Storage::vdisk_free($cfg, $volid);
6438                 }
6439                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6440             };
6441             print STDERR "unable to cleanup '$volid' - $@" if $@;
6442         }
6443         die $err;
6444     }
6445
6446     rmtree $tmpdir;
6447
6448     rename($tmpfn, $conffile) ||
6449         die "unable to commit configuration file '$conffile'\n";
6450
6451     PVE::Cluster::cfs_update(); # make sure we read new file
6452
6453     eval { rescan($vmid, 1); };
6454     warn $@ if $@;
6455 }
6456
6457 sub restore_tar_archive {
6458     my ($archive, $vmid, $user, $opts) = @_;
6459
6460     if ($archive ne '-') {
6461         my $firstfile = tar_archive_read_firstfile($archive);
6462         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6463             if $firstfile ne 'qemu-server.conf';
6464     }
6465
6466     my $storecfg = PVE::Storage::config();
6467
6468     # destroy existing data - keep empty config
6469     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6470     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6471
6472     my $tocmd = "/usr/lib/qemu-server/qmextract";
6473
6474     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6475     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6476     $tocmd .= ' --prealloc' if $opts->{prealloc};
6477     $tocmd .= ' --info' if $opts->{info};
6478
6479     # tar option "xf" does not autodetect compression when read from STDIN,
6480     # so we pipe to zcat
6481     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6482         PVE::Tools::shellquote("--to-command=$tocmd");
6483
6484     my $tmpdir = "/var/tmp/vzdumptmp$$";
6485     mkpath $tmpdir;
6486
6487     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6488     local $ENV{VZDUMP_VMID} = $vmid;
6489     local $ENV{VZDUMP_USER} = $user;
6490
6491     my $conffile = PVE::QemuConfig->config_file($vmid);
6492     my $tmpfn = "$conffile.$$.tmp";
6493
6494     # disable interrupts (always do cleanups)
6495     local $SIG{INT} =
6496         local $SIG{TERM} =
6497         local $SIG{QUIT} =
6498         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6499
6500     eval {
6501         # enable interrupts
6502         local $SIG{INT} =
6503             local $SIG{TERM} =
6504             local $SIG{QUIT} =
6505             local $SIG{HUP} =
6506             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6507
6508         if ($archive eq '-') {
6509             print "extracting archive from STDIN\n";
6510             run_command($cmd, input => "<&STDIN");
6511         } else {
6512             print "extracting archive '$archive'\n";
6513             run_command($cmd);
6514         }
6515
6516         return if $opts->{info};
6517
6518         # read new mapping
6519         my $map = {};
6520         my $statfile = "$tmpdir/qmrestore.stat";
6521         if (my $fd = IO::File->new($statfile, "r")) {
6522             while (defined (my $line = <$fd>)) {
6523                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6524                     $map->{$1} = $2 if $1;
6525                 } else {
6526                     print STDERR "unable to parse line in statfile - $line\n";
6527                 }
6528             }
6529             $fd->close();
6530         }
6531
6532         my $confsrc = "$tmpdir/qemu-server.conf";
6533
6534         my $srcfd = new IO::File($confsrc, "r") ||
6535             die "unable to open file '$confsrc'\n";
6536
6537         my $outfd = new IO::File ($tmpfn, "w") ||
6538             die "unable to write config for VM $vmid\n";
6539
6540         my $cookie = { netcount => 0 };
6541         while (defined (my $line = <$srcfd>)) {
6542             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6543         }
6544
6545         $srcfd->close();
6546         $outfd->close();
6547     };
6548     my $err = $@;
6549
6550     if ($err) {
6551
6552         unlink $tmpfn;
6553
6554         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6555
6556         die $err;
6557     }
6558
6559     rmtree $tmpdir;
6560
6561     rename $tmpfn, $conffile ||
6562         die "unable to commit configuration file '$conffile'\n";
6563
6564     PVE::Cluster::cfs_update(); # make sure we read new file
6565
6566     eval { rescan($vmid, 1); };
6567     warn $@ if $@;
6568 };
6569
6570 sub foreach_storage_used_by_vm {
6571     my ($conf, $func) = @_;
6572
6573     my $sidhash = {};
6574
6575     foreach_drive($conf, sub {
6576         my ($ds, $drive) = @_;
6577         return if drive_is_cdrom($drive);
6578
6579         my $volid = $drive->{file};
6580
6581         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6582         $sidhash->{$sid} = $sid if $sid;
6583     });
6584
6585     foreach my $sid (sort keys %$sidhash) {
6586         &$func($sid);
6587     }
6588 }
6589
6590 sub do_snapshots_with_qemu {
6591     my ($storecfg, $volid) = @_;
6592
6593     my $storage_name = PVE::Storage::parse_volume_id($volid);
6594
6595     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6596         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6597         return 1;
6598     }
6599
6600     if ($volid =~ m/\.(qcow2|qed)$/){
6601         return 1;
6602     }
6603
6604     return undef;
6605 }
6606
6607 sub qga_check_running {
6608     my ($vmid, $nowarn) = @_;
6609
6610     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6611     if ($@) {
6612         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6613         return 0;
6614     }
6615     return 1;
6616 }
6617
6618 sub template_create {
6619     my ($vmid, $conf, $disk) = @_;
6620
6621     my $storecfg = PVE::Storage::config();
6622
6623     foreach_drive($conf, sub {
6624         my ($ds, $drive) = @_;
6625
6626         return if drive_is_cdrom($drive);
6627         return if $disk && $ds ne $disk;
6628
6629         my $volid = $drive->{file};
6630         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6631
6632         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6633         $drive->{file} = $voliddst;
6634         $conf->{$ds} = print_drive($vmid, $drive);
6635         PVE::QemuConfig->write_config($vmid, $conf);
6636     });
6637 }
6638
6639 sub convert_iscsi_path {
6640     my ($path) = @_;
6641
6642     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6643         my $portal = $1;
6644         my $target = $2;
6645         my $lun = $3;
6646
6647         my $initiator_name = get_initiator_name();
6648
6649         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6650                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6651     }
6652
6653     die "cannot convert iscsi path '$path', unkown format\n";
6654 }
6655
6656 sub qemu_img_convert {
6657     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6658
6659     my $storecfg = PVE::Storage::config();
6660     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6661     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6662
6663     if ($src_storeid && $dst_storeid) {
6664
6665         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6666
6667         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6668         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6669
6670         my $src_format = qemu_img_format($src_scfg, $src_volname);
6671         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6672
6673         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6674         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6675
6676         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6677         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6678
6679         my $cmd = [];
6680         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6681         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6682         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6683         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6684
6685         if ($src_is_iscsi) {
6686             push @$cmd, '--image-opts';
6687             $src_path = convert_iscsi_path($src_path);
6688         } else {
6689             push @$cmd, '-f', $src_format;
6690         }
6691
6692         if ($dst_is_iscsi) {
6693             push @$cmd, '--target-image-opts';
6694             $dst_path = convert_iscsi_path($dst_path);
6695         } else {
6696             push @$cmd, '-O', $dst_format;
6697         }
6698
6699         push @$cmd, $src_path;
6700
6701         if (!$dst_is_iscsi && $is_zero_initialized) {
6702             push @$cmd, "zeroinit:$dst_path";
6703         } else {
6704             push @$cmd, $dst_path;
6705         }
6706
6707         my $parser = sub {
6708             my $line = shift;
6709             if($line =~ m/\((\S+)\/100\%\)/){
6710                 my $percent = $1;
6711                 my $transferred = int($size * $percent / 100);
6712                 my $remaining = $size - $transferred;
6713
6714                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6715             }
6716
6717         };
6718
6719         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6720         my $err = $@;
6721         die "copy failed: $err" if $err;
6722     }
6723 }
6724
6725 sub qemu_img_format {
6726     my ($scfg, $volname) = @_;
6727
6728     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6729         return $1;
6730     } else {
6731         return "raw";
6732     }
6733 }
6734
6735 sub qemu_drive_mirror {
6736     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6737
6738     $jobs = {} if !$jobs;
6739
6740     my $qemu_target;
6741     my $format;
6742     $jobs->{"drive-$drive"} = {};
6743
6744     if ($dst_volid =~ /^nbd:/) {
6745         $qemu_target = $dst_volid;
6746         $format = "nbd";
6747     } else {
6748         my $storecfg = PVE::Storage::config();
6749         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6750
6751         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6752
6753         $format = qemu_img_format($dst_scfg, $dst_volname);
6754
6755         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6756
6757         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6758     }
6759
6760     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6761     $opts->{format} = $format if $format;
6762
6763     print "drive mirror is starting for drive-$drive\n";
6764
6765     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6766
6767     if (my $err = $@) {
6768         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6769         die "mirroring error: $err";
6770     }
6771
6772     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6773 }
6774
6775 sub qemu_drive_mirror_monitor {
6776     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6777
6778     eval {
6779         my $err_complete = 0;
6780
6781         while (1) {
6782             die "storage migration timed out\n" if $err_complete > 300;
6783
6784             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6785
6786             my $running_mirror_jobs = {};
6787             foreach my $stat (@$stats) {
6788                 next if $stat->{type} ne 'mirror';
6789                 $running_mirror_jobs->{$stat->{device}} = $stat;
6790             }
6791
6792             my $readycounter = 0;
6793
6794             foreach my $job (keys %$jobs) {
6795
6796                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6797                     print "$job : finished\n";
6798                     delete $jobs->{$job};
6799                     next;
6800                 }
6801
6802                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6803
6804                 my $busy = $running_mirror_jobs->{$job}->{busy};
6805                 my $ready = $running_mirror_jobs->{$job}->{ready};
6806                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6807                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6808                     my $remaining = $total - $transferred;
6809                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6810
6811                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6812                 }
6813
6814                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6815             }
6816
6817             last if scalar(keys %$jobs) == 0;
6818
6819             if ($readycounter == scalar(keys %$jobs)) {
6820                 print "all mirroring jobs are ready \n";
6821                 last if $skipcomplete; #do the complete later
6822
6823                 if ($vmiddst && $vmiddst != $vmid) {
6824                     my $agent_running = $qga && qga_check_running($vmid);
6825                     if ($agent_running) {
6826                         print "freeze filesystem\n";
6827                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6828                     } else {
6829                         print "suspend vm\n";
6830                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6831                     }
6832
6833                     # if we clone a disk for a new target vm, we don't switch the disk
6834                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6835
6836                     if ($agent_running) {
6837                         print "unfreeze filesystem\n";
6838                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6839                     } else {
6840                         print "resume vm\n";
6841                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6842                     }
6843
6844                     last;
6845                 } else {
6846
6847                     foreach my $job (keys %$jobs) {
6848                         # try to switch the disk if source and destination are on the same guest
6849                         print "$job: Completing block job...\n";
6850
6851                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6852                         if ($@ =~ m/cannot be completed/) {
6853                             print "$job: Block job cannot be completed, try again.\n";
6854                             $err_complete++;
6855                         }else {
6856                             print "$job: Completed successfully.\n";
6857                             $jobs->{$job}->{complete} = 1;
6858                         }
6859                     }
6860                 }
6861             }
6862             sleep 1;
6863         }
6864     };
6865     my $err = $@;
6866
6867     if ($err) {
6868         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6869         die "mirroring error: $err";
6870     }
6871
6872 }
6873
6874 sub qemu_blockjobs_cancel {
6875     my ($vmid, $jobs) = @_;
6876
6877     foreach my $job (keys %$jobs) {
6878         print "$job: Cancelling block job\n";
6879         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6880         $jobs->{$job}->{cancel} = 1;
6881     }
6882
6883     while (1) {
6884         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6885
6886         my $running_jobs = {};
6887         foreach my $stat (@$stats) {
6888             $running_jobs->{$stat->{device}} = $stat;
6889         }
6890
6891         foreach my $job (keys %$jobs) {
6892
6893             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6894                 print "$job: Done.\n";
6895                 delete $jobs->{$job};
6896             }
6897         }
6898
6899         last if scalar(keys %$jobs) == 0;
6900
6901         sleep 1;
6902     }
6903 }
6904
6905 sub clone_disk {
6906     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6907         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6908
6909     my $newvolid;
6910
6911     if (!$full) {
6912         print "create linked clone of drive $drivename ($drive->{file})\n";
6913         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6914         push @$newvollist, $newvolid;
6915     } else {
6916
6917         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6918         $storeid = $storage if $storage;
6919
6920         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6921         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6922
6923         print "create full clone of drive $drivename ($drive->{file})\n";
6924         my $name = undef;
6925         if (drive_is_cloudinit($drive)) {
6926             $name = "vm-$newvmid-cloudinit";
6927             $snapname = undef;
6928             # cloudinit only supports raw and qcow2 atm:
6929             if ($dst_format eq 'qcow2') {
6930                 $name .= '.qcow2';
6931             } elsif ($dst_format ne 'raw') {
6932                 die "clone: unhandled format for cloudinit image\n";
6933             }
6934         }
6935         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6936         push @$newvollist, $newvolid;
6937
6938         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6939
6940         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6941         if (!$running || $snapname) {
6942             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6943         } else {
6944
6945             my $kvmver = get_running_qemu_version ($vmid);
6946             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6947                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6948                     if $drive->{iothread};
6949             }
6950
6951             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6952         }
6953     }
6954
6955     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6956
6957     my $disk = $drive;
6958     $disk->{format} = undef;
6959     $disk->{file} = $newvolid;
6960     $disk->{size} = $size;
6961
6962     return $disk;
6963 }
6964
6965 # this only works if VM is running
6966 sub get_current_qemu_machine {
6967     my ($vmid) = @_;
6968
6969     my $cmd = { execute => 'query-machines', arguments => {} };
6970     my $res = vm_qmp_command($vmid, $cmd);
6971
6972     my ($current, $default);
6973     foreach my $e (@$res) {
6974         $default = $e->{name} if $e->{'is-default'};
6975         $current = $e->{name} if $e->{'is-current'};
6976     }
6977
6978     # fallback to the default machine if current is not supported by qemu
6979     return $current || $default || 'pc';
6980 }
6981
6982 sub get_running_qemu_version {
6983     my ($vmid) = @_;
6984     my $cmd = { execute => 'query-version', arguments => {} };
6985     my $res = vm_qmp_command($vmid, $cmd);
6986     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6987 }
6988
6989 sub qemu_machine_feature_enabled {
6990     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6991
6992     my $current_major;
6993     my $current_minor;
6994
6995     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6996
6997         $current_major = $3;
6998         $current_minor = $4;
6999
7000     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7001
7002         $current_major = $1;
7003         $current_minor = $2;
7004     }
7005
7006     return 1 if $current_major > $version_major ||
7007                 ($current_major == $version_major &&
7008                  $current_minor >= $version_minor);
7009 }
7010
7011 sub qemu_machine_pxe {
7012     my ($vmid, $conf, $machine) = @_;
7013
7014     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
7015
7016     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7017         $machine .= '.pxe';
7018     }
7019
7020     return $machine;
7021 }
7022
7023 sub qemu_use_old_bios_files {
7024     my ($machine_type) = @_;
7025
7026     return if !$machine_type;
7027
7028     my $use_old_bios_files = undef;
7029
7030     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7031         $machine_type = $1;
7032         $use_old_bios_files = 1;
7033     } else {
7034         my $kvmver = kvm_user_version();
7035         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7036         # load new efi bios files on migration. So this hack is required to allow
7037         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7038         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7039         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7040     }
7041
7042     return ($use_old_bios_files, $machine_type);
7043 }
7044
7045 sub create_efidisk($$$$$) {
7046     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7047
7048     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7049     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7050
7051     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7052     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7053     PVE::Storage::activate_volumes($storecfg, [$volid]);
7054
7055     my $path = PVE::Storage::path($storecfg, $volid);
7056     eval {
7057         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7058     };
7059     die "Copying EFI vars image failed: $@" if $@;
7060
7061     return ($volid, $vars_size);
7062 }
7063
7064 sub vm_iothreads_list {
7065     my ($vmid) = @_;
7066
7067     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7068
7069     my $iothreads = {};
7070     foreach my $iothread (@$res) {
7071         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7072     }
7073
7074     return $iothreads;
7075 }
7076
7077 sub scsihw_infos {
7078     my ($conf, $drive) = @_;
7079
7080     my $maxdev = 0;
7081
7082     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7083         $maxdev = 7;
7084     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7085         $maxdev = 1;
7086     } else {
7087         $maxdev = 256;
7088     }
7089
7090     my $controller = int($drive->{index} / $maxdev);
7091     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7092
7093     return ($maxdev, $controller, $controller_prefix);
7094 }
7095
7096 sub add_hyperv_enlightenments {
7097     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7098
7099     return if $winversion < 6;
7100     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7101
7102     if ($gpu_passthrough || defined($hv_vendor_id)) {
7103         $hv_vendor_id //= 'proxmox';
7104         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7105     }
7106
7107     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7108         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7109         push @$cpuFlags , 'hv_vapic';
7110         push @$cpuFlags , 'hv_time';
7111     } else {
7112         push @$cpuFlags , 'hv_spinlocks=0xffff';
7113     }
7114
7115     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7116         push @$cpuFlags , 'hv_reset';
7117         push @$cpuFlags , 'hv_vpindex';
7118         push @$cpuFlags , 'hv_runtime';
7119     }
7120
7121     if ($winversion >= 7) {
7122         push @$cpuFlags , 'hv_relaxed';
7123
7124         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7125             push @$cpuFlags , 'hv_synic';
7126             push @$cpuFlags , 'hv_stimer';
7127         }
7128     }
7129 }
7130
7131 sub windows_version {
7132     my ($ostype) = @_;
7133
7134     return 0 if !$ostype;
7135
7136     my $winversion = 0;
7137
7138     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7139         $winversion = 5;
7140     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7141         $winversion = 6;
7142     } elsif ($ostype =~ m/^win(\d+)$/) {
7143         $winversion = $1;
7144     }
7145
7146     return $winversion;
7147 }
7148
7149 sub resolve_dst_disk_format {
7150         my ($storecfg, $storeid, $src_volname, $format) = @_;
7151         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7152
7153         if (!$format) {
7154             # if no target format is specified, use the source disk format as hint
7155             if ($src_volname) {
7156                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7157                 $format = qemu_img_format($scfg, $src_volname);
7158             } else {
7159                 return $defFormat;
7160             }
7161         }
7162
7163         # test if requested format is supported - else use default
7164         my $supported = grep { $_ eq $format } @$validFormats;
7165         $format = $defFormat if !$supported;
7166         return $format;
7167 }
7168
7169 sub resolve_first_disk {
7170     my $conf = shift;
7171     my @disks = PVE::QemuServer::valid_drive_names();
7172     my $firstdisk;
7173     foreach my $ds (reverse @disks) {
7174         next if !$conf->{$ds};
7175         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7176         next if PVE::QemuServer::drive_is_cdrom($disk);
7177         $firstdisk = $ds;
7178     }
7179     return $firstdisk;
7180 }
7181
7182 sub generate_uuid {
7183     my ($uuid, $uuid_str);
7184     UUID::generate($uuid);
7185     UUID::unparse($uuid, $uuid_str);
7186     return $uuid_str;
7187 }
7188
7189 sub generate_smbios1_uuid {
7190     return "uuid=".generate_uuid();
7191 }
7192
7193 sub nbd_stop {
7194     my ($vmid) = @_;
7195
7196     vm_mon_cmd($vmid, 'nbd-server-stop');
7197 }
7198
7199 # bash completion helper
7200
7201 sub complete_backup_archives {
7202     my ($cmdname, $pname, $cvalue) = @_;
7203
7204     my $cfg = PVE::Storage::config();
7205
7206     my $storeid;
7207
7208     if ($cvalue =~ m/^([^:]+):/) {
7209         $storeid = $1;
7210     }
7211
7212     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7213
7214     my $res = [];
7215     foreach my $id (keys %$data) {
7216         foreach my $item (@{$data->{$id}}) {
7217             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7218             push @$res, $item->{volid} if defined($item->{volid});
7219         }
7220     }
7221
7222     return $res;
7223 }
7224
7225 my $complete_vmid_full = sub {
7226     my ($running) = @_;
7227
7228     my $idlist = vmstatus();
7229
7230     my $res = [];
7231
7232     foreach my $id (keys %$idlist) {
7233         my $d = $idlist->{$id};
7234         if (defined($running)) {
7235             next if $d->{template};
7236             next if $running && $d->{status} ne 'running';
7237             next if !$running && $d->{status} eq 'running';
7238         }
7239         push @$res, $id;
7240
7241     }
7242     return $res;
7243 };
7244
7245 sub complete_vmid {
7246     return &$complete_vmid_full();
7247 }
7248
7249 sub complete_vmid_stopped {
7250     return &$complete_vmid_full(0);
7251 }
7252
7253 sub complete_vmid_running {
7254     return &$complete_vmid_full(1);
7255 }
7256
7257 sub complete_storage {
7258
7259     my $cfg = PVE::Storage::config();
7260     my $ids = $cfg->{ids};
7261
7262     my $res = [];
7263     foreach my $sid (keys %$ids) {
7264         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7265         next if !$ids->{$sid}->{content}->{images};
7266         push @$res, $sid;
7267     }
7268
7269     return $res;
7270 }
7271
7272 1;