PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::GuestHelpers;
  34 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  35 use PVE::QemuServer::Memory;
  36 use PVE::QemuServer::USB qw(parse_usb_device);
  37 use PVE::QemuServer::Cloudinit;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use Time::HiRes qw(gettimeofday);
  41 use File::Copy qw(copy);
  42 use URI::Escape;
  43
  44 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  45 my $OVMF = {
  46     x86_64 => [
  47         "$EDK2_FW_BASE/OVMF_CODE.fd",
  48         "$EDK2_FW_BASE/OVMF_VARS.fd"
  49     ],
  50     aarch64 => [
  51         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  52         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  53     ],
  54 };
  55
  56 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  57
  58 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  59
  60 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  61
  62 # Note about locking: we use flock on the config file protect
  63 # against concurent actions.
  64 # Aditionaly, we have a 'lock' setting in the config file. This
  65 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  66 # allowed when such lock is set. But you can ignore this kind of
  67 # lock with the --skiplock flag.
  68
  69 cfs_register_file('/qemu-server/',
  70                   \&parse_vm_config,
  71                   \&write_vm_config);
  72
  73 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  74     description => "Some command save/restore state from this location.",
  75     type => 'string',
  76     maxLength => 128,
  77     optional => 1,
  78 });
  79
  80 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  81     description => "The name of the snapshot.",
  82     type => 'string', format => 'pve-configid',
  83     maxLength => 40,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 mkdir "/etc/pve/nodes/$nodename";
 114 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 115 mkdir $confdir;
 116
 117 my $var_run_tmpdir = "/var/run/qemu-server";
 118 mkdir $var_run_tmpdir;
 119
 120 my $lock_dir = "/var/lock/qemu-server";
 121 mkdir $lock_dir;
 122
 123 my $cpu_vendor_list = {
 124     # Intel CPUs
 125     486 => 'GenuineIntel',
 126     pentium => 'GenuineIntel',
 127     pentium2  => 'GenuineIntel',
 128     pentium3  => 'GenuineIntel',
 129     coreduo => 'GenuineIntel',
 130     core2duo => 'GenuineIntel',
 131     Conroe  => 'GenuineIntel',
 132     Penryn  => 'GenuineIntel',
 133     Nehalem  => 'GenuineIntel',
 134     'Nehalem-IBRS'  => 'GenuineIntel',
 135     Westmere => 'GenuineIntel',
 136     'Westmere-IBRS' => 'GenuineIntel',
 137     SandyBridge => 'GenuineIntel',
 138     'SandyBridge-IBRS' => 'GenuineIntel',
 139     IvyBridge => 'GenuineIntel',
 140     'IvyBridge-IBRS' => 'GenuineIntel',
 141     Haswell => 'GenuineIntel',
 142     'Haswell-IBRS' => 'GenuineIntel',
 143     'Haswell-noTSX' => 'GenuineIntel',
 144     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 145     Broadwell => 'GenuineIntel',
 146     'Broadwell-IBRS' => 'GenuineIntel',
 147     'Broadwell-noTSX' => 'GenuineIntel',
 148     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 149     'Skylake-Client' => 'GenuineIntel',
 150     'Skylake-Client-IBRS' => 'GenuineIntel',
 151     'Skylake-Server' => 'GenuineIntel',
 152     'Skylake-Server-IBRS' => 'GenuineIntel',
 153
 154     # AMD CPUs
 155     athlon => 'AuthenticAMD',
 156     phenom  => 'AuthenticAMD',
 157     Opteron_G1  => 'AuthenticAMD',
 158     Opteron_G2  => 'AuthenticAMD',
 159     Opteron_G3  => 'AuthenticAMD',
 160     Opteron_G4  => 'AuthenticAMD',
 161     Opteron_G5  => 'AuthenticAMD',
 162     EPYC => 'AuthenticAMD',
 163     'EPYC-IBPB' => 'AuthenticAMD',
 164
 165     # generic types, use vendor from host node
 166     host => 'default',
 167     kvm32 => 'default',
 168     kvm64 => 'default',
 169     qemu32 => 'default',
 170     qemu64 => 'default',
 171     max => 'default',
 172 };
 173
 174 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 175
 176 my $cpu_fmt = {
 177     cputype => {
 178         description => "Emulated CPU type.",
 179         type => 'string',
 180         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 181         default => 'kvm64',
 182         default_key => 1,
 183     },
 184     hidden => {
 185         description => "Do not identify as a KVM virtual machine.",
 186         type => 'boolean',
 187         optional => 1,
 188         default => 0
 189     },
 190     'hv-vendor-id' => {
 191         type => 'string',
 192         pattern => qr/[a-zA-Z0-9]{1,12}/,
 193         format_description => 'vendor-id',
 194         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 195         optional => 1,
 196     },
 197     flags => {
 198         description => "List of additional CPU flags separated by ';'."
 199                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 200                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 201         format_description => '+FLAG[;-FLAG...]',
 202         type => 'string',
 203         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 204         optional => 1,
 205     },
 206 };
 207
 208 my $watchdog_fmt = {
 209     model => {
 210         default_key => 1,
 211         type => 'string',
 212         enum => [qw(i6300esb ib700)],
 213         description => "Watchdog type to emulate.",
 214         default => 'i6300esb',
 215         optional => 1,
 216     },
 217     action => {
 218         type => 'string',
 219         enum => [qw(reset shutdown poweroff pause debug none)],
 220         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 221         optional => 1,
 222     },
 223 };
 224 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 225
 226 my $agent_fmt = {
 227     enabled => {
 228         description => "Enable/disable Qemu GuestAgent.",
 229         type => 'boolean',
 230         default => 0,
 231         default_key => 1,
 232     },
 233     fstrim_cloned_disks => {
 234         description => "Run fstrim after cloning/moving a disk.",
 235         type => 'boolean',
 236         optional => 1,
 237         default => 0
 238     },
 239 };
 240
 241 my $vga_fmt = {
 242     type => {
 243         description => "Select the VGA type.",
 244         type => 'string',
 245         default => 'std',
 246         optional => 1,
 247         default_key => 1,
 248         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 249     },
 250     memory => {
 251         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 252         type => 'integer',
 253         optional => 1,
 254         minimum => 4,
 255         maximum => 512,
 256     },
 257 };
 258
 259 my $ivshmem_fmt = {
 260     size => {
 261         type => 'integer',
 262         minimum => 1,
 263         description => "The size of the file in MB.",
 264     },
 265     name => {
 266         type => 'string',
 267         pattern => '[a-zA-Z0-9\-]+',
 268         optional => 1,
 269         format_description => 'string',
 270         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6/3.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string',
 396         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 397         pattern => '[acdn]{1,4}',
 398         default => 'cdn',
 399     },
 400     bootdisk => {
 401         optional => 1,
 402         type => 'string', format => 'pve-qm-bootdisk',
 403         description => "Enable booting from specified disk.",
 404         pattern => '(ide|sata|scsi|virtio)\d+',
 405     },
 406     smp => {
 407         optional => 1,
 408         type => 'integer',
 409         description => "The number of CPUs. Please use option -sockets instead.",
 410         minimum => 1,
 411         default => 1,
 412     },
 413     sockets => {
 414         optional => 1,
 415         type => 'integer',
 416         description => "The number of CPU sockets.",
 417         minimum => 1,
 418         default => 1,
 419     },
 420     cores => {
 421         optional => 1,
 422         type => 'integer',
 423         description => "The number of cores per socket.",
 424         minimum => 1,
 425         default => 1,
 426     },
 427     numa => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Enable/disable NUMA.",
 431         default => 0,
 432     },
 433     hugepages => {
 434         optional => 1,
 435         type => 'string',
 436         description => "Enable/disable hugepages memory.",
 437         enum => [qw(any 2 1024)],
 438     },
 439     vcpus => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Number of hotplugged vcpus.",
 443         minimum => 1,
 444         default => 0,
 445     },
 446     acpi => {
 447         optional => 1,
 448         type => 'boolean',
 449         description => "Enable/disable ACPI.",
 450         default => 1,
 451     },
 452     agent => {
 453         optional => 1,
 454         description => "Enable/disable Qemu GuestAgent and its properties.",
 455         type => 'string',
 456         format => $agent_fmt,
 457     },
 458     kvm => {
 459         optional => 1,
 460         type => 'boolean',
 461         description => "Enable/disable KVM hardware virtualization.",
 462         default => 1,
 463     },
 464     tdf => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable time drift fix.",
 468         default => 0,
 469     },
 470     localtime => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 474     },
 475     freeze => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 479     },
 480     vga => {
 481         optional => 1,
 482         type => 'string', format => $vga_fmt,
 483         description => "Configure the VGA hardware.",
 484         verbose_description => "Configure the VGA Hardware. If you want to use ".
 485             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 486             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 487             "is 'std' for all OS types besides some Windows versions (XP and " .
 488             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 489             "display server. For win* OS you can select how many independent " .
 490             "displays you want, Linux guests can add displays them self.\n".
 491             "You can also run without any graphic card, using a serial device as terminal.",
 492     },
 493     watchdog => {
 494         optional => 1,
 495         type => 'string', format => 'pve-qm-watchdog',
 496         description => "Create a virtual hardware watchdog device.",
 497         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 498             " (by a guest action), the watchdog must be periodically polled " .
 499             "by an agent inside the guest or else the watchdog will reset " .
 500             "the guest (or execute the respective action specified)",
 501     },
 502     startdate => {
 503         optional => 1,
 504         type => 'string',
 505         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 506         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 507         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 508         default => 'now',
 509     },
 510     startup =>  get_standard_option('pve-startup-order'),
 511     template => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Enable/disable Template.",
 515         default => 0,
 516     },
 517     args => {
 518         optional => 1,
 519         type => 'string',
 520         description => "Arbitrary arguments passed to kvm.",
 521         verbose_description => <<EODESCR,
 522 Arbitrary arguments passed to kvm, for example:
 523
 524 args: -no-reboot -no-hpet
 525
 526 NOTE: this option is for experts only.
 527 EODESCR
 528     },
 529     tablet => {
 530         optional => 1,
 531         type => 'boolean',
 532         default => 1,
 533         description => "Enable/disable the USB tablet device.",
 534         verbose_description => "Enable/disable the USB tablet device. This device is " .
 535             "usually needed to allow absolute mouse positioning with VNC. " .
 536             "Else the mouse runs out of sync with normal VNC clients. " .
 537             "If you're running lots of console-only guests on one host, " .
 538             "you may consider disabling this to save some context switches. " .
 539             "This is turned off by default if you use spice (-vga=qxl).",
 540     },
 541     migrate_speed => {
 542         optional => 1,
 543         type => 'integer',
 544         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 545         minimum => 0,
 546         default => 0,
 547     },
 548     migrate_downtime => {
 549         optional => 1,
 550         type => 'number',
 551         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 552         minimum => 0,
 553         default => 0.1,
 554     },
 555     cdrom => {
 556         optional => 1,
 557         type => 'string', format => 'pve-qm-ide',
 558         typetext => '<volume>',
 559         description => "This is an alias for option -ide2",
 560     },
 561     cpu => {
 562         optional => 1,
 563         description => "Emulated CPU type.",
 564         type => 'string',
 565         format => $cpu_fmt,
 566     },
 567     parent => get_standard_option('pve-snapshot-name', {
 568         optional => 1,
 569         description => "Parent snapshot name. This is used internally, and should not be modified.",
 570     }),
 571     snaptime => {
 572         optional => 1,
 573         description => "Timestamp for snapshots.",
 574         type => 'integer',
 575         minimum => 0,
 576     },
 577     vmstate => {
 578         optional => 1,
 579         type => 'string', format => 'pve-volume-id',
 580         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 581     },
 582     vmstatestorage => get_standard_option('pve-storage-id', {
 583         description => "Default storage for VM state volumes/files.",
 584         optional => 1,
 585     }),
 586     runningmachine => get_standard_option('pve-qemu-machine', {
 587         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 588     }),
 589     machine => get_standard_option('pve-qemu-machine'),
 590     arch => {
 591         description => "Virtual processor architecture. Defaults to the host.",
 592         optional => 1,
 593         type => 'string',
 594         enum => [qw(x86_64 aarch64)],
 595     },
 596     smbios1 => {
 597         description => "Specify SMBIOS type 1 fields.",
 598         type => 'string', format => 'pve-qm-smbios1',
 599         maxLength => 256,
 600         optional => 1,
 601     },
 602     protection => {
 603         optional => 1,
 604         type => 'boolean',
 605         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 606         default => 0,
 607     },
 608     bios => {
 609         optional => 1,
 610         type => 'string',
 611         enum => [ qw(seabios ovmf) ],
 612         description => "Select BIOS implementation.",
 613         default => 'seabios',
 614     },
 615     vmgenid => {
 616         type => 'string',
 617         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 618         format_description => 'UUID',
 619         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 620         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 621             " 128-bit integer value identifier to the guest OS. This allows to".
 622             " notify the guest operating system when the virtual machine is".
 623             " executed with a different configuration (e.g. snapshot execution".
 624             " or creation from a template). The guest operating system notices".
 625             " the change, and is then able to react as appropriate by marking".
 626             " its copies of distributed databases as dirty, re-initializing its".
 627             " random number generator, etc.\n".
 628             "Note that auto-creation only works when done throug API/CLI create".
 629             " or update methods, but not when manually editing the config file.",
 630         default => "1 (autogenerated)",
 631         optional => 1,
 632     },
 633     hookscript => {
 634         type => 'string',
 635         format => 'pve-volume-id',
 636         optional => 1,
 637         description => "Script that will be executed during various steps in the vms lifetime.",
 638     },
 639     ivshmem => {
 640         type => 'string',
 641         format => $ivshmem_fmt,
 642         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 643         optional => 1,
 644     }
 645 };
 646
 647 my $cicustom_fmt = {
 648     meta => {
 649         type => 'string',
 650         optional => 1,
 651         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 652         format => 'pve-volume-id',
 653         format_description => 'volume',
 654     },
 655     network => {
 656         type => 'string',
 657         optional => 1,
 658         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 659         format => 'pve-volume-id',
 660         format_description => 'volume',
 661     },
 662     user => {
 663         type => 'string',
 664         optional => 1,
 665         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 666         format => 'pve-volume-id',
 667         format_description => 'volume',
 668     },
 669 };
 670 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 671
 672 my $confdesc_cloudinit = {
 673     citype => {
 674         optional => 1,
 675         type => 'string',
 676         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 677         enum => ['configdrive2', 'nocloud'],
 678     },
 679     ciuser => {
 680         optional => 1,
 681         type => 'string',
 682         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 683     },
 684     cipassword => {
 685         optional => 1,
 686         type => 'string',
 687         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 688     },
 689     cicustom => {
 690         optional => 1,
 691         type => 'string',
 692         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 693         format => 'pve-qm-cicustom',
 694     },
 695     searchdomain => {
 696         optional => 1,
 697         type => 'string',
 698         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 699     },
 700     nameserver => {
 701         optional => 1,
 702         type => 'string', format => 'address-list',
 703         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 704     },
 705     sshkeys => {
 706         optional => 1,
 707         type => 'string',
 708         format => 'urlencoded',
 709         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 710     },
 711 };
 712
 713 # what about other qemu settings ?
 714 #cpu => 'string',
 715 #machine => 'string',
 716 #fda => 'file',
 717 #fdb => 'file',
 718 #mtdblock => 'file',
 719 #sd => 'file',
 720 #pflash => 'file',
 721 #snapshot => 'bool',
 722 #bootp => 'file',
 723 ##tftp => 'dir',
 724 ##smb => 'dir',
 725 #kernel => 'file',
 726 #append => 'string',
 727 #initrd => 'file',
 728 ##soundhw => 'string',
 729
 730 while (my ($k, $v) = each %$confdesc) {
 731     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 732 }
 733
 734 my $MAX_IDE_DISKS = 4;
 735 my $MAX_SCSI_DISKS = 14;
 736 my $MAX_VIRTIO_DISKS = 16;
 737 my $MAX_SATA_DISKS = 6;
 738 my $MAX_USB_DEVICES = 5;
 739 my $MAX_NETS = 32;
 740 my $MAX_UNUSED_DISKS = 256;
 741 my $MAX_HOSTPCI_DEVICES = 4;
 742 my $MAX_SERIAL_PORTS = 4;
 743 my $MAX_PARALLEL_PORTS = 3;
 744 my $MAX_NUMA = 8;
 745
 746 my $numa_fmt = {
 747     cpus => {
 748         type => "string",
 749         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 750         description => "CPUs accessing this NUMA node.",
 751         format_description => "id[-id];...",
 752     },
 753     memory => {
 754         type => "number",
 755         description => "Amount of memory this NUMA node provides.",
 756         optional => 1,
 757     },
 758     hostnodes => {
 759         type => "string",
 760         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 761         description => "Host NUMA nodes to use.",
 762         format_description => "id[-id];...",
 763         optional => 1,
 764     },
 765     policy => {
 766         type => 'string',
 767         enum => [qw(preferred bind interleave)],
 768         description => "NUMA allocation policy.",
 769         optional => 1,
 770     },
 771 };
 772 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 773 my $numadesc = {
 774     optional => 1,
 775     type => 'string', format => $numa_fmt,
 776     description => "NUMA topology.",
 777 };
 778 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 779
 780 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 781     $confdesc->{"numa$i"} = $numadesc;
 782 }
 783
 784 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 785                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 786                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 787 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 788
 789 my $net_fmt_bridge_descr = <<__EOD__;
 790 Bridge to attach the network device to. The Proxmox VE standard bridge
 791 is called 'vmbr0'.
 792
 793 If you do not specify a bridge, we create a kvm user (NATed) network
 794 device, which provides DHCP and DNS services. The following addresses
 795 are used:
 796
 797  10.0.2.2   Gateway
 798  10.0.2.3   DNS Server
 799  10.0.2.4   SMB Server
 800
 801 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 802 __EOD__
 803
 804 my $net_fmt = {
 805     macaddr  => get_standard_option('mac-addr', {
 806         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 807     }),
 808     model => {
 809         type => 'string',
 810         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 811         enum => $nic_model_list,
 812         default_key => 1,
 813     },
 814     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 815     bridge => {
 816         type => 'string',
 817         description => $net_fmt_bridge_descr,
 818         format_description => 'bridge',
 819         optional => 1,
 820     },
 821     queues => {
 822         type => 'integer',
 823         minimum => 0, maximum => 16,
 824         description => 'Number of packet queues to be used on the device.',
 825         optional => 1,
 826     },
 827     rate => {
 828         type => 'number',
 829         minimum => 0,
 830         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 831         optional => 1,
 832     },
 833     tag => {
 834         type => 'integer',
 835         minimum => 1, maximum => 4094,
 836         description => 'VLAN tag to apply to packets on this interface.',
 837         optional => 1,
 838     },
 839     trunks => {
 840         type => 'string',
 841         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 842         description => 'VLAN trunks to pass through this interface.',
 843         format_description => 'vlanid[;vlanid...]',
 844         optional => 1,
 845     },
 846     firewall => {
 847         type => 'boolean',
 848         description => 'Whether this interface should be protected by the firewall.',
 849         optional => 1,
 850     },
 851     link_down => {
 852         type => 'boolean',
 853         description => 'Whether this interface should be disconnected (like pulling the plug).',
 854         optional => 1,
 855     },
 856 };
 857
 858 my $netdesc = {
 859     optional => 1,
 860     type => 'string', format => $net_fmt,
 861     description => "Specify network devices.",
 862 };
 863
 864 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 865
 866 my $ipconfig_fmt = {
 867     ip => {
 868         type => 'string',
 869         format => 'pve-ipv4-config',
 870         format_description => 'IPv4Format/CIDR',
 871         description => 'IPv4 address in CIDR format.',
 872         optional => 1,
 873         default => 'dhcp',
 874     },
 875     gw => {
 876         type => 'string',
 877         format => 'ipv4',
 878         format_description => 'GatewayIPv4',
 879         description => 'Default gateway for IPv4 traffic.',
 880         optional => 1,
 881         requires => 'ip',
 882     },
 883     ip6 => {
 884         type => 'string',
 885         format => 'pve-ipv6-config',
 886         format_description => 'IPv6Format/CIDR',
 887         description => 'IPv6 address in CIDR format.',
 888         optional => 1,
 889         default => 'dhcp',
 890     },
 891     gw6 => {
 892         type => 'string',
 893         format => 'ipv6',
 894         format_description => 'GatewayIPv6',
 895         description => 'Default gateway for IPv6 traffic.',
 896         optional => 1,
 897         requires => 'ip6',
 898     },
 899 };
 900 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 901 my $ipconfigdesc = {
 902     optional => 1,
 903     type => 'string', format => 'pve-qm-ipconfig',
 904     description => <<'EODESCR',
 905 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 906
 907 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 908
 909 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 910 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 911
 912 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 913 EODESCR
 914 };
 915 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 916
 917 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 918     $confdesc->{"net$i"} = $netdesc;
 919     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 920 }
 921
 922 foreach my $key (keys %$confdesc_cloudinit) {
 923     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 924 }
 925
 926 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 927 sub verify_volume_id_or_qm_path {
 928     my ($volid, $noerr) = @_;
 929
 930     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 931         return $volid;
 932     }
 933
 934     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 935     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 936     if ($@) {
 937         return undef if $noerr;
 938         die $@;
 939     }
 940     return $volid;
 941 }
 942
 943 my $drivename_hash;
 944
 945 my %drivedesc_base = (
 946     volume => { alias => 'file' },
 947     file => {
 948         type => 'string',
 949         format => 'pve-volume-id-or-qm-path',
 950         default_key => 1,
 951         format_description => 'volume',
 952         description => "The drive's backing volume.",
 953     },
 954     media => {
 955         type => 'string',
 956         enum => [qw(cdrom disk)],
 957         description => "The drive's media type.",
 958         default => 'disk',
 959         optional => 1
 960     },
 961     cyls => {
 962         type => 'integer',
 963         description => "Force the drive's physical geometry to have a specific cylinder count.",
 964         optional => 1
 965     },
 966     heads => {
 967         type => 'integer',
 968         description => "Force the drive's physical geometry to have a specific head count.",
 969         optional => 1
 970     },
 971     secs => {
 972         type => 'integer',
 973         description => "Force the drive's physical geometry to have a specific sector count.",
 974         optional => 1
 975     },
 976     trans => {
 977         type => 'string',
 978         enum => [qw(none lba auto)],
 979         description => "Force disk geometry bios translation mode.",
 980         optional => 1,
 981     },
 982     snapshot => {
 983         type => 'boolean',
 984         description => "Controls qemu's snapshot mode feature."
 985             . " If activated, changes made to the disk are temporary and will"
 986             . " be discarded when the VM is shutdown.",
 987         optional => 1,
 988     },
 989     cache => {
 990         type => 'string',
 991         enum => [qw(none writethrough writeback unsafe directsync)],
 992         description => "The drive's cache mode",
 993         optional => 1,
 994     },
 995     format => get_standard_option('pve-qm-image-format'),
 996     size => {
 997         type => 'string',
 998         format => 'disk-size',
 999         format_description => 'DiskSize',
1000         description => "Disk size. This is purely informational and has no effect.",
1001         optional => 1,
1002     },
1003     backup => {
1004         type => 'boolean',
1005         description => "Whether the drive should be included when making backups.",
1006         optional => 1,
1007     },
1008     replicate => {
1009         type => 'boolean',
1010         description => 'Whether the drive should considered for replication jobs.',
1011         optional => 1,
1012         default => 1,
1013     },
1014     rerror => {
1015         type => 'string',
1016         enum => [qw(ignore report stop)],
1017         description => 'Read error action.',
1018         optional => 1,
1019     },
1020     werror => {
1021         type => 'string',
1022         enum => [qw(enospc ignore report stop)],
1023         description => 'Write error action.',
1024         optional => 1,
1025     },
1026     aio => {
1027         type => 'string',
1028         enum => [qw(native threads)],
1029         description => 'AIO type to use.',
1030         optional => 1,
1031     },
1032     discard => {
1033         type => 'string',
1034         enum => [qw(ignore on)],
1035         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1036         optional => 1,
1037     },
1038     detect_zeroes => {
1039         type => 'boolean',
1040         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1041         optional => 1,
1042     },
1043     serial => {
1044         type => 'string',
1045         format => 'urlencoded',
1046         format_description => 'serial',
1047         maxLength => 20*3, # *3 since it's %xx url enoded
1048         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1049         optional => 1,
1050     },
1051     shared => {
1052         type => 'boolean',
1053         description => 'Mark this locally-managed volume as available on all nodes',
1054         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1055         optional => 1,
1056         default => 0,
1057     }
1058 );
1059
1060 my %iothread_fmt = ( iothread => {
1061         type => 'boolean',
1062         description => "Whether to use iothreads for this drive",
1063         optional => 1,
1064 });
1065
1066 my %model_fmt = (
1067     model => {
1068         type => 'string',
1069         format => 'urlencoded',
1070         format_description => 'model',
1071         maxLength => 40*3, # *3 since it's %xx url enoded
1072         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1073         optional => 1,
1074     },
1075 );
1076
1077 my %queues_fmt = (
1078     queues => {
1079         type => 'integer',
1080         description => "Number of queues.",
1081         minimum => 2,
1082         optional => 1
1083     }
1084 );
1085
1086 my %scsiblock_fmt = (
1087     scsiblock => {
1088         type => 'boolean',
1089         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1090         optional => 1,
1091         default => 0,
1092     },
1093 );
1094
1095 my %ssd_fmt = (
1096     ssd => {
1097         type => 'boolean',
1098         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1099         optional => 1,
1100     },
1101 );
1102
1103 my %wwn_fmt = (
1104     wwn => {
1105         type => 'string',
1106         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1107         format_description => 'wwn',
1108         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1109         optional => 1,
1110     },
1111 );
1112
1113 my $add_throttle_desc = sub {
1114     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1115     my $d = {
1116         type => $type,
1117         format_description => $unit,
1118         description => "Maximum $what in $longunit.",
1119         optional => 1,
1120     };
1121     $d->{minimum} = $minimum if defined($minimum);
1122     $drivedesc_base{$key} = $d;
1123 };
1124 # throughput: (leaky bucket)
1125 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1126 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1127 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1128 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1129 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1130 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1131 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1132 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1133 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1134
1135 # pools: (pool of IO before throttling starts taking effect)
1136 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1137 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1138 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1139 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1140 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1141 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1142
1143 # burst lengths
1144 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1145 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1146 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1147 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1148 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1149 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1150
1151 # legacy support
1152 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1153 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1154 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1155 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1156
1157 my $ide_fmt = {
1158     %drivedesc_base,
1159     %model_fmt,
1160     %ssd_fmt,
1161     %wwn_fmt,
1162 };
1163 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1164
1165 my $idedesc = {
1166     optional => 1,
1167     type => 'string', format => $ide_fmt,
1168     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1169 };
1170 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1171
1172 my $scsi_fmt = {
1173     %drivedesc_base,
1174     %iothread_fmt,
1175     %queues_fmt,
1176     %scsiblock_fmt,
1177     %ssd_fmt,
1178     %wwn_fmt,
1179 };
1180 my $scsidesc = {
1181     optional => 1,
1182     type => 'string', format => $scsi_fmt,
1183     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1184 };
1185 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1186
1187 my $sata_fmt = {
1188     %drivedesc_base,
1189     %ssd_fmt,
1190     %wwn_fmt,
1191 };
1192 my $satadesc = {
1193     optional => 1,
1194     type => 'string', format => $sata_fmt,
1195     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1196 };
1197 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1198
1199 my $virtio_fmt = {
1200     %drivedesc_base,
1201     %iothread_fmt,
1202 };
1203 my $virtiodesc = {
1204     optional => 1,
1205     type => 'string', format => $virtio_fmt,
1206     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1207 };
1208 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1209
1210 my $alldrive_fmt = {
1211     %drivedesc_base,
1212     %iothread_fmt,
1213     %model_fmt,
1214     %queues_fmt,
1215     %scsiblock_fmt,
1216     %ssd_fmt,
1217     %wwn_fmt,
1218 };
1219
1220 my $efidisk_fmt = {
1221     volume => { alias => 'file' },
1222     file => {
1223         type => 'string',
1224         format => 'pve-volume-id-or-qm-path',
1225         default_key => 1,
1226         format_description => 'volume',
1227         description => "The drive's backing volume.",
1228     },
1229     format => get_standard_option('pve-qm-image-format'),
1230     size => {
1231         type => 'string',
1232         format => 'disk-size',
1233         format_description => 'DiskSize',
1234         description => "Disk size. This is purely informational and has no effect.",
1235         optional => 1,
1236     },
1237 };
1238
1239 my $efidisk_desc = {
1240     optional => 1,
1241     type => 'string', format => $efidisk_fmt,
1242     description => "Configure a Disk for storing EFI vars",
1243 };
1244
1245 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1246
1247 my $usb_fmt = {
1248     host => {
1249         default_key => 1,
1250         type => 'string', format => 'pve-qm-usb-device',
1251         format_description => 'HOSTUSBDEVICE|spice',
1252         description => <<EODESCR,
1253 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1254
1255  'bus-port(.port)*' (decimal numbers) or
1256  'vendor_id:product_id' (hexadeciaml numbers) or
1257  'spice'
1258
1259 You can use the 'lsusb -t' command to list existing usb devices.
1260
1261 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1262
1263 The value 'spice' can be used to add a usb redirection devices for spice.
1264 EODESCR
1265     },
1266     usb3 => {
1267         optional => 1,
1268         type => 'boolean',
1269         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1270         default => 0,
1271     },
1272 };
1273
1274 my $usbdesc = {
1275     optional => 1,
1276     type => 'string', format => $usb_fmt,
1277     description => "Configure an USB device (n is 0 to 4).",
1278 };
1279 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1280
1281 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1282 my $hostpci_fmt = {
1283     host => {
1284         default_key => 1,
1285         type => 'string',
1286         pattern => qr/$PCIRE(;$PCIRE)*/,
1287         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1288         description => <<EODESCR,
1289 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1290 of PCI virtual functions of the host. HOSTPCIID syntax is:
1291
1292 'bus:dev.func' (hexadecimal numbers)
1293
1294 You can us the 'lspci' command to list existing PCI devices.
1295 EODESCR
1296     },
1297     rombar => {
1298         type => 'boolean',
1299         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1300         optional => 1,
1301         default => 1,
1302     },
1303     romfile => {
1304         type => 'string',
1305         pattern => '[^,;]+',
1306         format_description => 'string',
1307         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1308         optional => 1,
1309     },
1310     pcie => {
1311         type => 'boolean',
1312         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1313         optional => 1,
1314         default => 0,
1315     },
1316     'x-vga' => {
1317         type => 'boolean',
1318         description =>  "Enable vfio-vga device support.",
1319         optional => 1,
1320         default => 0,
1321     },
1322     'mdev' => {
1323         type => 'string',
1324         format_description => 'string',
1325         pattern => '[^/\.:]+',
1326         optional => 1,
1327         description => <<EODESCR
1328 The type of mediated device to use.
1329 An instance of this type will be created on startup of the VM and
1330 will be cleaned up when the VM stops.
1331 EODESCR
1332     }
1333 };
1334 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1335
1336 my $hostpcidesc = {
1337         optional => 1,
1338         type => 'string', format => 'pve-qm-hostpci',
1339         description => "Map host PCI devices into guest.",
1340         verbose_description =>  <<EODESCR,
1341 Map host PCI devices into guest.
1342
1343 NOTE: This option allows direct access to host hardware. So it is no longer
1344 possible to migrate such machines - use with special care.
1345
1346 CAUTION: Experimental! User reported problems with this option.
1347 EODESCR
1348 };
1349 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1350
1351 my $serialdesc = {
1352         optional => 1,
1353         type => 'string',
1354         pattern => '(/dev/.+|socket)',
1355         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1356         verbose_description =>  <<EODESCR,
1357 Create a serial device inside the VM (n is 0 to 3), and pass through a
1358 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1359 host side (use 'qm terminal' to open a terminal connection).
1360
1361 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1362
1363 CAUTION: Experimental! User reported problems with this option.
1364 EODESCR
1365 };
1366
1367 my $paralleldesc= {
1368         optional => 1,
1369         type => 'string',
1370         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1371         description =>  "Map host parallel devices (n is 0 to 2).",
1372         verbose_description =>  <<EODESCR,
1373 Map host parallel devices (n is 0 to 2).
1374
1375 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1376
1377 CAUTION: Experimental! User reported problems with this option.
1378 EODESCR
1379 };
1380
1381 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1382     $confdesc->{"parallel$i"} = $paralleldesc;
1383 }
1384
1385 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1386     $confdesc->{"serial$i"} = $serialdesc;
1387 }
1388
1389 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1390     $confdesc->{"hostpci$i"} = $hostpcidesc;
1391 }
1392
1393 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1394     $drivename_hash->{"ide$i"} = 1;
1395     $confdesc->{"ide$i"} = $idedesc;
1396 }
1397
1398 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1399     $drivename_hash->{"sata$i"} = 1;
1400     $confdesc->{"sata$i"} = $satadesc;
1401 }
1402
1403 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1404     $drivename_hash->{"scsi$i"} = 1;
1405     $confdesc->{"scsi$i"} = $scsidesc ;
1406 }
1407
1408 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1409     $drivename_hash->{"virtio$i"} = 1;
1410     $confdesc->{"virtio$i"} = $virtiodesc;
1411 }
1412
1413 $drivename_hash->{efidisk0} = 1;
1414 $confdesc->{efidisk0} = $efidisk_desc;
1415
1416 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1417     $confdesc->{"usb$i"} = $usbdesc;
1418 }
1419
1420 my $unuseddesc = {
1421     optional => 1,
1422     type => 'string', format => 'pve-volume-id',
1423     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1424 };
1425
1426 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1427     $confdesc->{"unused$i"} = $unuseddesc;
1428 }
1429
1430 my $kvm_api_version = 0;
1431
1432 sub kvm_version {
1433     return $kvm_api_version if $kvm_api_version;
1434
1435     open my $fh, '<', '/dev/kvm'
1436         or return undef;
1437
1438     # 0xae00 => KVM_GET_API_VERSION
1439     $kvm_api_version = ioctl($fh, 0xae00, 0);
1440
1441     return $kvm_api_version;
1442 }
1443
1444 my $kvm_user_version;
1445
1446 sub kvm_user_version {
1447
1448     return $kvm_user_version if $kvm_user_version;
1449
1450     $kvm_user_version = 'unknown';
1451
1452     my $code = sub {
1453         my $line = shift;
1454         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1455             $kvm_user_version = $2;
1456         }
1457     };
1458
1459     eval { run_command("kvm -version", outfunc => $code); };
1460     warn $@ if $@;
1461
1462     return $kvm_user_version;
1463
1464 }
1465
1466 sub kernel_has_vhost_net {
1467     return -c '/dev/vhost-net';
1468 }
1469
1470 sub valid_drive_names {
1471     # order is important - used to autoselect boot disk
1472     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1473             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1474             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1475             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1476             'efidisk0');
1477 }
1478
1479 sub is_valid_drivename {
1480     my $dev = shift;
1481
1482     return defined($drivename_hash->{$dev});
1483 }
1484
1485 sub option_exists {
1486     my $key = shift;
1487     return defined($confdesc->{$key});
1488 }
1489
1490 sub nic_models {
1491     return $nic_model_list;
1492 }
1493
1494 sub os_list_description {
1495
1496     return {
1497         other => 'Other',
1498         wxp => 'Windows XP',
1499         w2k => 'Windows 2000',
1500         w2k3 =>, 'Windows 2003',
1501         w2k8 => 'Windows 2008',
1502         wvista => 'Windows Vista',
1503         win7 => 'Windows 7',
1504         win8 => 'Windows 8/2012',
1505         win10 => 'Windows 10/2016',
1506         l24 => 'Linux 2.4',
1507         l26 => 'Linux 2.6',
1508     };
1509 }
1510
1511 my $cdrom_path;
1512
1513 sub get_cdrom_path {
1514
1515     return  $cdrom_path if $cdrom_path;
1516
1517     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1518     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1519     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1520 }
1521
1522 sub get_iso_path {
1523     my ($storecfg, $vmid, $cdrom) = @_;
1524
1525     if ($cdrom eq 'cdrom') {
1526         return get_cdrom_path();
1527     } elsif ($cdrom eq 'none') {
1528         return '';
1529     } elsif ($cdrom =~ m|^/|) {
1530         return $cdrom;
1531     } else {
1532         return PVE::Storage::path($storecfg, $cdrom);
1533     }
1534 }
1535
1536 # try to convert old style file names to volume IDs
1537 sub filename_to_volume_id {
1538     my ($vmid, $file, $media) = @_;
1539
1540      if (!($file eq 'none' || $file eq 'cdrom' ||
1541           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1542
1543         return undef if $file =~ m|/|;
1544
1545         if ($media && $media eq 'cdrom') {
1546             $file = "local:iso/$file";
1547         } else {
1548             $file = "local:$vmid/$file";
1549         }
1550     }
1551
1552     return $file;
1553 }
1554
1555 sub verify_media_type {
1556     my ($opt, $vtype, $media) = @_;
1557
1558     return if !$media;
1559
1560     my $etype;
1561     if ($media eq 'disk') {
1562         $etype = 'images';
1563     } elsif ($media eq 'cdrom') {
1564         $etype = 'iso';
1565     } else {
1566         die "internal error";
1567     }
1568
1569     return if ($vtype eq $etype);
1570
1571     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1572 }
1573
1574 sub cleanup_drive_path {
1575     my ($opt, $storecfg, $drive) = @_;
1576
1577     # try to convert filesystem paths to volume IDs
1578
1579     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1580         ($drive->{file} !~ m|^/dev/.+|) &&
1581         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1582         ($drive->{file} !~ m/^\d+$/)) {
1583         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1584         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1585         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1586         verify_media_type($opt, $vtype, $drive->{media});
1587         $drive->{file} = $volid;
1588     }
1589
1590     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1591 }
1592
1593 sub parse_hotplug_features {
1594     my ($data) = @_;
1595
1596     my $res = {};
1597
1598     return $res if $data eq '0';
1599
1600     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1601
1602     foreach my $feature (PVE::Tools::split_list($data)) {
1603         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1604             $res->{$1} = 1;
1605         } else {
1606             die "invalid hotplug feature '$feature'\n";
1607         }
1608     }
1609     return $res;
1610 }
1611
1612 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1613 sub pve_verify_hotplug_features {
1614     my ($value, $noerr) = @_;
1615
1616     return $value if parse_hotplug_features($value);
1617
1618     return undef if $noerr;
1619
1620     die "unable to parse hotplug option\n";
1621 }
1622
1623 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1624 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1625 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1626 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1627 #        [,iothread=on][,serial=serial][,model=model]
1628
1629 sub parse_drive {
1630     my ($key, $data) = @_;
1631
1632     my ($interface, $index);
1633
1634     if ($key =~ m/^([^\d]+)(\d+)$/) {
1635         $interface = $1;
1636         $index = $2;
1637     } else {
1638         return undef;
1639     }
1640
1641     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1642                                      : $confdesc->{$key}->{format};
1643     if (!$desc) {
1644         warn "invalid drive key: $key\n";
1645         return undef;
1646     }
1647     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1648     return undef if !$res;
1649     $res->{interface} = $interface;
1650     $res->{index} = $index;
1651
1652     my $error = 0;
1653     foreach my $opt (qw(bps bps_rd bps_wr)) {
1654         if (my $bps = defined(delete $res->{$opt})) {
1655             if (defined($res->{"m$opt"})) {
1656                 warn "both $opt and m$opt specified\n";
1657                 ++$error;
1658                 next;
1659             }
1660             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1661         }
1662     }
1663
1664     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1665     for my $requirement (
1666         [mbps_max => 'mbps'],
1667         [mbps_rd_max => 'mbps_rd'],
1668         [mbps_wr_max => 'mbps_wr'],
1669         [miops_max => 'miops'],
1670         [miops_rd_max => 'miops_rd'],
1671         [miops_wr_max => 'miops_wr'],
1672         [bps_max_length => 'mbps_max'],
1673         [bps_rd_max_length => 'mbps_rd_max'],
1674         [bps_wr_max_length => 'mbps_wr_max'],
1675         [iops_max_length => 'iops_max'],
1676         [iops_rd_max_length => 'iops_rd_max'],
1677         [iops_wr_max_length => 'iops_wr_max']) {
1678         my ($option, $requires) = @$requirement;
1679         if ($res->{$option} && !$res->{$requires}) {
1680             warn "$option requires $requires\n";
1681             ++$error;
1682         }
1683     }
1684
1685     return undef if $error;
1686
1687     return undef if $res->{mbps_rd} && $res->{mbps};
1688     return undef if $res->{mbps_wr} && $res->{mbps};
1689     return undef if $res->{iops_rd} && $res->{iops};
1690     return undef if $res->{iops_wr} && $res->{iops};
1691
1692     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1693         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1694         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1695         return undef if $res->{interface} eq 'virtio';
1696     }
1697
1698     if (my $size = $res->{size}) {
1699         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1700     }
1701
1702     return $res;
1703 }
1704
1705 sub print_drive {
1706     my ($vmid, $drive) = @_;
1707     my $data = { %$drive };
1708     delete $data->{$_} for qw(index interface);
1709     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1710 }
1711
1712 sub scsi_inquiry {
1713     my($fh, $noerr) = @_;
1714
1715     my $SG_IO = 0x2285;
1716     my $SG_GET_VERSION_NUM = 0x2282;
1717
1718     my $versionbuf = "\x00" x 8;
1719     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1720     if (!$ret) {
1721         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1722         return undef;
1723     }
1724     my $version = unpack("I", $versionbuf);
1725     if ($version < 30000) {
1726         die "scsi generic interface too old\n"  if !$noerr;
1727         return undef;
1728     }
1729
1730     my $buf = "\x00" x 36;
1731     my $sensebuf = "\x00" x 8;
1732     my $cmd = pack("C x3 C x1", 0x12, 36);
1733
1734     # see /usr/include/scsi/sg.h
1735     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1736
1737     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1738                       length($sensebuf), 0, length($buf), $buf,
1739                       $cmd, $sensebuf, 6000);
1740
1741     $ret = ioctl($fh, $SG_IO, $packet);
1742     if (!$ret) {
1743         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1744         return undef;
1745     }
1746
1747     my @res = unpack($sg_io_hdr_t, $packet);
1748     if ($res[17] || $res[18]) {
1749         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1750         return undef;
1751     }
1752
1753     my $res = {};
1754     (my $byte0, my $byte1, $res->{vendor},
1755      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1756
1757     $res->{removable} = $byte1 & 128 ? 1 : 0;
1758     $res->{type} = $byte0 & 31;
1759
1760     return $res;
1761 }
1762
1763 sub path_is_scsi {
1764     my ($path) = @_;
1765
1766     my $fh = IO::File->new("+<$path") || return undef;
1767     my $res = scsi_inquiry($fh, 1);
1768     close($fh);
1769
1770     return $res;
1771 }
1772
1773 sub machine_type_is_q35 {
1774     my ($conf) = @_;
1775
1776     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1777 }
1778
1779 sub print_tabletdevice_full {
1780     my ($conf, $arch) = @_;
1781
1782     my $q35 = machine_type_is_q35($conf);
1783
1784     # we use uhci for old VMs because tablet driver was buggy in older qemu
1785     my $usbbus;
1786     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1787         $usbbus = 'ehci';
1788     } else {
1789         $usbbus = 'uhci';
1790     }
1791
1792     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1793 }
1794
1795 sub print_keyboarddevice_full {
1796     my ($conf, $arch, $machine) = @_;
1797
1798     return undef if $arch ne 'aarch64';
1799
1800     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1801 }
1802
1803 sub print_drivedevice_full {
1804     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1805
1806     my $device = '';
1807     my $maxdev = 0;
1808
1809     if ($drive->{interface} eq 'virtio') {
1810         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1811         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1812         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1813     } elsif ($drive->{interface} eq 'scsi') {
1814
1815         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1816         my $unit = $drive->{index} % $maxdev;
1817         my $devicetype = 'hd';
1818         my $path = '';
1819         if (drive_is_cdrom($drive)) {
1820             $devicetype = 'cd';
1821         } else {
1822             if ($drive->{file} =~ m|^/|) {
1823                 $path = $drive->{file};
1824                 if (my $info = path_is_scsi($path)) {
1825                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1826                         $devicetype = 'block';
1827                     } elsif ($info->{type} == 1) { # tape
1828                         $devicetype = 'generic';
1829                     }
1830                 }
1831             } else {
1832                  $path = PVE::Storage::path($storecfg, $drive->{file});
1833             }
1834
1835             if($path =~ m/^iscsi\:\/\//){
1836                 $devicetype = 'generic';
1837             }
1838         }
1839
1840         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1841             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1842         } else {
1843             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1844         }
1845
1846         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1847             $device .= ",rotation_rate=1";
1848         }
1849         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1850
1851     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1852         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1853         my $controller = int($drive->{index} / $maxdev);
1854         my $unit = $drive->{index} % $maxdev;
1855         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1856
1857         $device = "ide-$devicetype";
1858         if ($drive->{interface} eq 'ide') {
1859             $device .= ",bus=ide.$controller,unit=$unit";
1860         } else {
1861             $device .= ",bus=ahci$controller.$unit";
1862         }
1863         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1864
1865         if ($devicetype eq 'hd') {
1866             if (my $model = $drive->{model}) {
1867                 $model = URI::Escape::uri_unescape($model);
1868                 $device .= ",model=$model";
1869             }
1870             if ($drive->{ssd}) {
1871                 $device .= ",rotation_rate=1";
1872             }
1873         }
1874         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1875     } elsif ($drive->{interface} eq 'usb') {
1876         die "implement me";
1877         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1878     } else {
1879         die "unsupported interface type";
1880     }
1881
1882     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1883
1884     if (my $serial = $drive->{serial}) {
1885         $serial = URI::Escape::uri_unescape($serial);
1886         $device .= ",serial=$serial";
1887     }
1888
1889
1890     return $device;
1891 }
1892
1893 sub get_initiator_name {
1894     my $initiator;
1895
1896     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1897     while (defined(my $line = <$fh>)) {
1898         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1899         $initiator = $1;
1900         last;
1901     }
1902     $fh->close();
1903
1904     return $initiator;
1905 }
1906
1907 sub print_drive_full {
1908     my ($storecfg, $vmid, $drive) = @_;
1909
1910     my $path;
1911     my $volid = $drive->{file};
1912     my $format;
1913
1914     if (drive_is_cdrom($drive)) {
1915         $path = get_iso_path($storecfg, $vmid, $volid);
1916     } else {
1917         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1918         if ($storeid) {
1919             $path = PVE::Storage::path($storecfg, $volid);
1920             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1921             $format = qemu_img_format($scfg, $volname);
1922         } else {
1923             $path = $volid;
1924             $format = "raw";
1925         }
1926    }
1927
1928     my $opts = '';
1929     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1930     foreach my $o (@qemu_drive_options) {
1931         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1932     }
1933
1934     # snapshot only accepts on|off
1935     if (defined($drive->{snapshot})) {
1936         my $v = $drive->{snapshot} ? 'on' : 'off';
1937         $opts .= ",snapshot=$v";
1938     }
1939
1940     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1941         my ($dir, $qmpname) = @$type;
1942         if (my $v = $drive->{"mbps$dir"}) {
1943             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1944         }
1945         if (my $v = $drive->{"mbps${dir}_max"}) {
1946             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1947         }
1948         if (my $v = $drive->{"bps${dir}_max_length"}) {
1949             $opts .= ",throttling.bps$qmpname-max-length=$v";
1950         }
1951         if (my $v = $drive->{"iops${dir}"}) {
1952             $opts .= ",throttling.iops$qmpname=$v";
1953         }
1954         if (my $v = $drive->{"iops${dir}_max"}) {
1955             $opts .= ",throttling.iops$qmpname-max=$v";
1956         }
1957         if (my $v = $drive->{"iops${dir}_max_length"}) {
1958             $opts .= ",throttling.iops$qmpname-max-length=$v";
1959         }
1960     }
1961
1962     $opts .= ",format=$format" if $format && !$drive->{format};
1963
1964     my $cache_direct = 0;
1965
1966     if (my $cache = $drive->{cache}) {
1967         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1968     } elsif (!drive_is_cdrom($drive)) {
1969         $opts .= ",cache=none";
1970         $cache_direct = 1;
1971     }
1972
1973     # aio native works only with O_DIRECT
1974     if (!$drive->{aio}) {
1975         if($cache_direct) {
1976             $opts .= ",aio=native";
1977         } else {
1978             $opts .= ",aio=threads";
1979         }
1980     }
1981
1982     if (!drive_is_cdrom($drive)) {
1983         my $detectzeroes;
1984         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1985             $detectzeroes = 'off';
1986         } elsif ($drive->{discard}) {
1987             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1988         } else {
1989             # This used to be our default with discard not being specified:
1990             $detectzeroes = 'on';
1991         }
1992         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1993     }
1994
1995     my $pathinfo = $path ? "file=$path," : '';
1996
1997     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1998 }
1999
2000 sub print_netdevice_full {
2001     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2002
2003     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2004
2005     my $device = $net->{model};
2006     if ($net->{model} eq 'virtio') {
2007          $device = 'virtio-net-pci';
2008      };
2009
2010     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2011     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2012     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2013         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2014         my $vectors = $net->{queues} * 2 + 2;
2015         $tmpstr .= ",vectors=$vectors,mq=on";
2016     }
2017     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2018
2019     if ($use_old_bios_files) {
2020         my $romfile;
2021         if ($device eq 'virtio-net-pci') {
2022             $romfile = 'pxe-virtio.rom';
2023         } elsif ($device eq 'e1000') {
2024             $romfile = 'pxe-e1000.rom';
2025         } elsif ($device eq 'ne2k') {
2026             $romfile = 'pxe-ne2k_pci.rom';
2027         } elsif ($device eq 'pcnet') {
2028             $romfile = 'pxe-pcnet.rom';
2029         } elsif ($device eq 'rtl8139') {
2030             $romfile = 'pxe-rtl8139.rom';
2031         }
2032         $tmpstr .= ",romfile=$romfile" if $romfile;
2033     }
2034
2035     return $tmpstr;
2036 }
2037
2038 sub print_netdev_full {
2039     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2040
2041     my $i = '';
2042     if ($netid =~ m/^net(\d+)$/) {
2043         $i = int($1);
2044     }
2045
2046     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2047
2048     my $ifname = "tap${vmid}i$i";
2049
2050     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2051     die "interface name '$ifname' is too long (max 15 character)\n"
2052         if length($ifname) >= 16;
2053
2054     my $vhostparam = '';
2055     if (is_native($arch)) {
2056         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2057     }
2058
2059     my $vmname = $conf->{name} || "vm$vmid";
2060
2061     my $netdev = "";
2062     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2063
2064     if ($net->{bridge}) {
2065         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2066     } else {
2067         $netdev = "type=user,id=$netid,hostname=$vmname";
2068     }
2069
2070     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2071
2072     return $netdev;
2073 }
2074
2075
2076 sub print_cpu_device {
2077     my ($conf, $id) = @_;
2078
2079     my $kvm = $conf->{kvm} // 1;
2080     my $cpu = $kvm ? "kvm64" : "qemu64";
2081     if (my $cputype = $conf->{cpu}) {
2082         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2083             or die "Cannot parse cpu description: $cputype\n";
2084         $cpu = $cpuconf->{cputype};
2085     }
2086
2087     my $cores = $conf->{cores} || 1;
2088
2089     my $current_core = ($id - 1) % $cores;
2090     my $current_socket = int(($id - 1 - $current_core)/$cores);
2091
2092     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2093 }
2094
2095 my $vga_map = {
2096     'cirrus' => 'cirrus-vga',
2097     'std' => 'VGA',
2098     'vmware' => 'vmware-svga',
2099     'virtio' => 'virtio-vga',
2100 };
2101
2102 sub print_vga_device {
2103     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2104
2105     my $type = $vga_map->{$vga->{type}};
2106     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2107         $type = 'virtio-gpu';
2108     }
2109     my $vgamem_mb = $vga->{memory};
2110     if ($qxlnum) {
2111         $type = $id ? 'qxl' : 'qxl-vga';
2112     }
2113     die "no devicetype for $vga->{type}\n" if !$type;
2114
2115     my $memory = "";
2116     if ($vgamem_mb) {
2117         if ($vga->{type} eq 'virtio') {
2118             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2119             $memory = ",max_hostmem=$bytes";
2120         } elsif ($qxlnum) {
2121             # from https://www.spice-space.org/multiple-monitors.html
2122             $memory = ",vgamem_mb=$vga->{memory}";
2123             my $ram = $vgamem_mb * 4;
2124             my $vram = $vgamem_mb * 2;
2125             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2126         } else {
2127             $memory = ",vgamem_mb=$vga->{memory}";
2128         }
2129     } elsif ($qxlnum && $id) {
2130         $memory = ",ram_size=67108864,vram_size=33554432";
2131     }
2132
2133     my $q35 = machine_type_is_q35($conf);
2134     my $vgaid = "vga" . ($id // '');
2135     my $pciaddr;
2136
2137     if ($q35 && $vgaid eq 'vga') {
2138         # the first display uses pcie.0 bus on q35 machines
2139         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2140     } else {
2141         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2142     }
2143
2144     return "$type,id=${vgaid}${memory}${pciaddr}";
2145 }
2146
2147 sub drive_is_cloudinit {
2148     my ($drive) = @_;
2149     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2150 }
2151
2152 sub drive_is_cdrom {
2153     my ($drive, $exclude_cloudinit) = @_;
2154
2155     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2156
2157     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2158
2159 }
2160
2161 sub parse_number_sets {
2162     my ($set) = @_;
2163     my $res = [];
2164     foreach my $part (split(/;/, $set)) {
2165         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2166             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2167             push @$res, [ $1, $2 ];
2168         } else {
2169             die "invalid range: $part\n";
2170         }
2171     }
2172     return $res;
2173 }
2174
2175 sub parse_numa {
2176     my ($data) = @_;
2177
2178     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2179     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2180     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2181     return $res;
2182 }
2183
2184 sub parse_hostpci {
2185     my ($value) = @_;
2186
2187     return undef if !$value;
2188
2189     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2190
2191     my @idlist = split(/;/, $res->{host});
2192     delete $res->{host};
2193     foreach my $id (@idlist) {
2194         if ($id =~ m/\./) { # full id 00:00.1
2195             push @{$res->{pciid}}, {
2196                 id => $id,
2197             };
2198         } else { # partial id 00:00
2199             $res->{pciid} = PVE::SysFSTools::lspci($id);
2200         }
2201     }
2202     return $res;
2203 }
2204
2205 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2206 sub parse_net {
2207     my ($data) = @_;
2208
2209     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2210     if ($@) {
2211         warn $@;
2212         return undef;
2213     }
2214     if (!defined($res->{macaddr})) {
2215         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2216         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2217     }
2218     return $res;
2219 }
2220
2221 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2222 sub parse_ipconfig {
2223     my ($data) = @_;
2224
2225     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2226     if ($@) {
2227         warn $@;
2228         return undef;
2229     }
2230
2231     if ($res->{gw} && !$res->{ip}) {
2232         warn 'gateway specified without specifying an IP address';
2233         return undef;
2234     }
2235     if ($res->{gw6} && !$res->{ip6}) {
2236         warn 'IPv6 gateway specified without specifying an IPv6 address';
2237         return undef;
2238     }
2239     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2240         warn 'gateway specified together with DHCP';
2241         return undef;
2242     }
2243     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2244         # gw6 + auto/dhcp
2245         warn "IPv6 gateway specified together with $res->{ip6} address";
2246         return undef;
2247     }
2248
2249     if (!$res->{ip} && !$res->{ip6}) {
2250         return { ip => 'dhcp', ip6 => 'dhcp' };
2251     }
2252
2253     return $res;
2254 }
2255
2256 sub print_net {
2257     my $net = shift;
2258
2259     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2260 }
2261
2262 sub add_random_macs {
2263     my ($settings) = @_;
2264
2265     foreach my $opt (keys %$settings) {
2266         next if $opt !~ m/^net(\d+)$/;
2267         my $net = parse_net($settings->{$opt});
2268         next if !$net;
2269         $settings->{$opt} = print_net($net);
2270     }
2271 }
2272
2273 sub vm_is_volid_owner {
2274     my ($storecfg, $vmid, $volid) = @_;
2275
2276     if ($volid !~  m|^/|) {
2277         my ($path, $owner);
2278         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2279         if ($owner && ($owner == $vmid)) {
2280             return 1;
2281         }
2282     }
2283
2284     return undef;
2285 }
2286
2287 sub split_flagged_list {
2288     my $text = shift || '';
2289     $text =~ s/[,;]/ /g;
2290     $text =~ s/^\s+//;
2291     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2292 }
2293
2294 sub join_flagged_list {
2295     my ($how, $lst) = @_;
2296     join $how, map { $lst->{$_} . $_ } keys %$lst;
2297 }
2298
2299 sub vmconfig_delete_pending_option {
2300     my ($conf, $key, $force) = @_;
2301
2302     delete $conf->{pending}->{$key};
2303     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2304     $pending_delete_hash->{$key} = $force ? '!' : '';
2305     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2306 }
2307
2308 sub vmconfig_undelete_pending_option {
2309     my ($conf, $key) = @_;
2310
2311     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2312     delete $pending_delete_hash->{$key};
2313
2314     if (%$pending_delete_hash) {
2315         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2316     } else {
2317         delete $conf->{pending}->{delete};
2318     }
2319 }
2320
2321 sub vmconfig_register_unused_drive {
2322     my ($storecfg, $vmid, $conf, $drive) = @_;
2323
2324     if (drive_is_cloudinit($drive)) {
2325         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2326         warn $@ if $@;
2327     } elsif (!drive_is_cdrom($drive)) {
2328         my $volid = $drive->{file};
2329         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2330             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2331         }
2332     }
2333 }
2334
2335 sub vmconfig_cleanup_pending {
2336     my ($conf) = @_;
2337
2338     # remove pending changes when nothing changed
2339     my $changes;
2340     foreach my $opt (keys %{$conf->{pending}}) {
2341         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2342             $changes = 1;
2343             delete $conf->{pending}->{$opt};
2344         }
2345     }
2346
2347     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2348     my $pending_delete_hash = {};
2349     while (my ($opt, $force) = each %$current_delete_hash) {
2350         if (defined($conf->{$opt})) {
2351             $pending_delete_hash->{$opt} = $force;
2352         } else {
2353             $changes = 1;
2354         }
2355     }
2356
2357     if (%$pending_delete_hash) {
2358         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2359     } else {
2360         delete $conf->{pending}->{delete};
2361     }
2362
2363     return $changes;
2364 }
2365
2366 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2367 my $smbios1_fmt = {
2368     uuid => {
2369         type => 'string',
2370         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2371         format_description => 'UUID',
2372         description => "Set SMBIOS1 UUID.",
2373         optional => 1,
2374     },
2375     version => {
2376         type => 'string',
2377         pattern => '\S+',
2378         format_description => 'string',
2379         description => "Set SMBIOS1 version.",
2380         optional => 1,
2381     },
2382     serial => {
2383         type => 'string',
2384         pattern => '\S+',
2385         format_description => 'string',
2386         description => "Set SMBIOS1 serial number.",
2387         optional => 1,
2388     },
2389     manufacturer => {
2390         type => 'string',
2391         pattern => '\S+',
2392         format_description => 'string',
2393         description => "Set SMBIOS1 manufacturer.",
2394         optional => 1,
2395     },
2396     product => {
2397         type => 'string',
2398         pattern => '\S+',
2399         format_description => 'string',
2400         description => "Set SMBIOS1 product ID.",
2401         optional => 1,
2402     },
2403     sku => {
2404         type => 'string',
2405         pattern => '\S+',
2406         format_description => 'string',
2407         description => "Set SMBIOS1 SKU string.",
2408         optional => 1,
2409     },
2410     family => {
2411         type => 'string',
2412         pattern => '\S+',
2413         format_description => 'string',
2414         description => "Set SMBIOS1 family string.",
2415         optional => 1,
2416     },
2417 };
2418
2419 sub parse_smbios1 {
2420     my ($data) = @_;
2421
2422     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2423     warn $@ if $@;
2424     return $res;
2425 }
2426
2427 sub print_smbios1 {
2428     my ($smbios1) = @_;
2429     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2430 }
2431
2432 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2433
2434 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2435 sub verify_bootdisk {
2436     my ($value, $noerr) = @_;
2437
2438     return $value if is_valid_drivename($value);
2439
2440     return undef if $noerr;
2441
2442     die "invalid boot disk '$value'\n";
2443 }
2444
2445 sub parse_watchdog {
2446     my ($value) = @_;
2447
2448     return undef if !$value;
2449
2450     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2451     warn $@ if $@;
2452     return $res;
2453 }
2454
2455 sub parse_guest_agent {
2456     my ($value) = @_;
2457
2458     return {} if !defined($value->{agent});
2459
2460     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2461     warn $@ if $@;
2462
2463     # if the agent is disabled ignore the other potentially set properties
2464     return {} if !$res->{enabled};
2465     return $res;
2466 }
2467
2468 sub parse_vga {
2469     my ($value) = @_;
2470
2471     return {} if !$value;
2472     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2473     warn $@ if $@;
2474     return $res;
2475 }
2476
2477 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2478 sub verify_usb_device {
2479     my ($value, $noerr) = @_;
2480
2481     return $value if parse_usb_device($value);
2482
2483     return undef if $noerr;
2484
2485     die "unable to parse usb device\n";
2486 }
2487
2488 # add JSON properties for create and set function
2489 sub json_config_properties {
2490     my $prop = shift;
2491
2492     foreach my $opt (keys %$confdesc) {
2493         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2494         $prop->{$opt} = $confdesc->{$opt};
2495     }
2496
2497     return $prop;
2498 }
2499
2500 # return copy of $confdesc_cloudinit to generate documentation
2501 sub cloudinit_config_properties {
2502
2503     return dclone($confdesc_cloudinit);
2504 }
2505
2506 sub check_type {
2507     my ($key, $value) = @_;
2508
2509     die "unknown setting '$key'\n" if !$confdesc->{$key};
2510
2511     my $type = $confdesc->{$key}->{type};
2512
2513     if (!defined($value)) {
2514         die "got undefined value\n";
2515     }
2516
2517     if ($value =~ m/[\n\r]/) {
2518         die "property contains a line feed\n";
2519     }
2520
2521     if ($type eq 'boolean') {
2522         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2523         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2524         die "type check ('boolean') failed - got '$value'\n";
2525     } elsif ($type eq 'integer') {
2526         return int($1) if $value =~ m/^(\d+)$/;
2527         die "type check ('integer') failed - got '$value'\n";
2528     } elsif ($type eq 'number') {
2529         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2530         die "type check ('number') failed - got '$value'\n";
2531     } elsif ($type eq 'string') {
2532         if (my $fmt = $confdesc->{$key}->{format}) {
2533             PVE::JSONSchema::check_format($fmt, $value);
2534             return $value;
2535         }
2536         $value =~ s/^\"(.*)\"$/$1/;
2537         return $value;
2538     } else {
2539         die "internal error"
2540     }
2541 }
2542
2543 sub touch_config {
2544     my ($vmid) = @_;
2545
2546     my $conf = PVE::QemuConfig->config_file($vmid);
2547     utime undef, undef, $conf;
2548 }
2549
2550 sub destroy_vm {
2551     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2552
2553     my $conffile = PVE::QemuConfig->config_file($vmid);
2554
2555     my $conf = PVE::QemuConfig->load_config($vmid);
2556
2557     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2558
2559     if ($conf->{template}) {
2560         # check if any base image is still used by a linked clone
2561         foreach_drive($conf, sub {
2562                 my ($ds, $drive) = @_;
2563
2564                 return if drive_is_cdrom($drive);
2565
2566                 my $volid = $drive->{file};
2567
2568                 return if !$volid || $volid =~ m|^/|;
2569
2570                 die "base volume '$volid' is still in use by linked cloned\n"
2571                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2572
2573         });
2574     }
2575
2576     # only remove disks owned by this VM
2577     foreach_drive($conf, sub {
2578         my ($ds, $drive) = @_;
2579
2580         return if drive_is_cdrom($drive, 1);
2581
2582         my $volid = $drive->{file};
2583
2584         return if !$volid || $volid =~ m|^/|;
2585
2586         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2587         return if !$path || !$owner || ($owner != $vmid);
2588
2589         eval {
2590             PVE::Storage::vdisk_free($storecfg, $volid);
2591         };
2592         warn "Could not remove disk '$volid', check manually: $@" if $@;
2593
2594     });
2595
2596     if ($keep_empty_config) {
2597         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2598     } else {
2599         unlink $conffile;
2600     }
2601
2602     # also remove unused disk
2603     eval {
2604         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2605
2606         eval {
2607             PVE::Storage::foreach_volid($dl, sub {
2608                 my ($volid, $sid, $volname, $d) = @_;
2609                 PVE::Storage::vdisk_free($storecfg, $volid);
2610             });
2611         };
2612         warn $@ if $@;
2613
2614     };
2615     warn $@ if $@;
2616 }
2617
2618 sub parse_vm_config {
2619     my ($filename, $raw) = @_;
2620
2621     return undef if !defined($raw);
2622
2623     my $res = {
2624         digest => Digest::SHA::sha1_hex($raw),
2625         snapshots => {},
2626         pending => {},
2627     };
2628
2629     $filename =~ m|/qemu-server/(\d+)\.conf$|
2630         || die "got strange filename '$filename'";
2631
2632     my $vmid = $1;
2633
2634     my $conf = $res;
2635     my $descr;
2636     my $section = '';
2637
2638     my @lines = split(/\n/, $raw);
2639     foreach my $line (@lines) {
2640         next if $line =~ m/^\s*$/;
2641
2642         if ($line =~ m/^\[PENDING\]\s*$/i) {
2643             $section = 'pending';
2644             if (defined($descr)) {
2645                 $descr =~ s/\s+$//;
2646                 $conf->{description} = $descr;
2647             }
2648             $descr = undef;
2649             $conf = $res->{$section} = {};
2650             next;
2651
2652         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2653             $section = $1;
2654             if (defined($descr)) {
2655                 $descr =~ s/\s+$//;
2656                 $conf->{description} = $descr;
2657             }
2658             $descr = undef;
2659             $conf = $res->{snapshots}->{$section} = {};
2660             next;
2661         }
2662
2663         if ($line =~ m/^\#(.*)\s*$/) {
2664             $descr = '' if !defined($descr);
2665             $descr .= PVE::Tools::decode_text($1) . "\n";
2666             next;
2667         }
2668
2669         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2670             $descr = '' if !defined($descr);
2671             $descr .= PVE::Tools::decode_text($2);
2672         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2673             $conf->{snapstate} = $1;
2674         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2675             my $key = $1;
2676             my $value = $2;
2677             $conf->{$key} = $value;
2678         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2679             my $value = $1;
2680             if ($section eq 'pending') {
2681                 $conf->{delete} = $value; # we parse this later
2682             } else {
2683                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2684             }
2685         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2686             my $key = $1;
2687             my $value = $2;
2688             eval { $value = check_type($key, $value); };
2689             if ($@) {
2690                 warn "vm $vmid - unable to parse value of '$key' - $@";
2691             } else {
2692                 $key = 'ide2' if $key eq 'cdrom';
2693                 my $fmt = $confdesc->{$key}->{format};
2694                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2695                     my $v = parse_drive($key, $value);
2696                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2697                         $v->{file} = $volid;
2698                         $value = print_drive($vmid, $v);
2699                     } else {
2700                         warn "vm $vmid - unable to parse value of '$key'\n";
2701                         next;
2702                     }
2703                 }
2704
2705                 $conf->{$key} = $value;
2706             }
2707         }
2708     }
2709
2710     if (defined($descr)) {
2711         $descr =~ s/\s+$//;
2712         $conf->{description} = $descr;
2713     }
2714     delete $res->{snapstate}; # just to be sure
2715
2716     return $res;
2717 }
2718
2719 sub write_vm_config {
2720     my ($filename, $conf) = @_;
2721
2722     delete $conf->{snapstate}; # just to be sure
2723
2724     if ($conf->{cdrom}) {
2725         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2726         $conf->{ide2} = $conf->{cdrom};
2727         delete $conf->{cdrom};
2728     }
2729
2730     # we do not use 'smp' any longer
2731     if ($conf->{sockets}) {
2732         delete $conf->{smp};
2733     } elsif ($conf->{smp}) {
2734         $conf->{sockets} = $conf->{smp};
2735         delete $conf->{cores};
2736         delete $conf->{smp};
2737     }
2738
2739     my $used_volids = {};
2740
2741     my $cleanup_config = sub {
2742         my ($cref, $pending, $snapname) = @_;
2743
2744         foreach my $key (keys %$cref) {
2745             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2746                 $key eq 'snapstate' || $key eq 'pending';
2747             my $value = $cref->{$key};
2748             if ($key eq 'delete') {
2749                 die "propertry 'delete' is only allowed in [PENDING]\n"
2750                     if !$pending;
2751                 # fixme: check syntax?
2752                 next;
2753             }
2754             eval { $value = check_type($key, $value); };
2755             die "unable to parse value of '$key' - $@" if $@;
2756
2757             $cref->{$key} = $value;
2758
2759             if (!$snapname && is_valid_drivename($key)) {
2760                 my $drive = parse_drive($key, $value);
2761                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2762             }
2763         }
2764     };
2765
2766     &$cleanup_config($conf);
2767
2768     &$cleanup_config($conf->{pending}, 1);
2769
2770     foreach my $snapname (keys %{$conf->{snapshots}}) {
2771         die "internal error" if $snapname eq 'pending';
2772         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2773     }
2774
2775     # remove 'unusedX' settings if we re-add a volume
2776     foreach my $key (keys %$conf) {
2777         my $value = $conf->{$key};
2778         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2779             delete $conf->{$key};
2780         }
2781     }
2782
2783     my $generate_raw_config = sub {
2784         my ($conf, $pending) = @_;
2785
2786         my $raw = '';
2787
2788         # add description as comment to top of file
2789         if (defined(my $descr = $conf->{description})) {
2790             if ($descr) {
2791                 foreach my $cl (split(/\n/, $descr)) {
2792                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2793                 }
2794             } else {
2795                 $raw .= "#\n" if $pending;
2796             }
2797         }
2798
2799         foreach my $key (sort keys %$conf) {
2800             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2801             $raw .= "$key: $conf->{$key}\n";
2802         }
2803         return $raw;
2804     };
2805
2806     my $raw = &$generate_raw_config($conf);
2807
2808     if (scalar(keys %{$conf->{pending}})){
2809         $raw .= "\n[PENDING]\n";
2810         $raw .= &$generate_raw_config($conf->{pending}, 1);
2811     }
2812
2813     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2814         $raw .= "\n[$snapname]\n";
2815         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2816     }
2817
2818     return $raw;
2819 }
2820
2821 sub load_defaults {
2822
2823     my $res = {};
2824
2825     # we use static defaults from our JSON schema configuration
2826     foreach my $key (keys %$confdesc) {
2827         if (defined(my $default = $confdesc->{$key}->{default})) {
2828             $res->{$key} = $default;
2829         }
2830     }
2831
2832     return $res;
2833 }
2834
2835 sub config_list {
2836     my $vmlist = PVE::Cluster::get_vmlist();
2837     my $res = {};
2838     return $res if !$vmlist || !$vmlist->{ids};
2839     my $ids = $vmlist->{ids};
2840
2841     foreach my $vmid (keys %$ids) {
2842         my $d = $ids->{$vmid};
2843         next if !$d->{node} || $d->{node} ne $nodename;
2844         next if !$d->{type} || $d->{type} ne 'qemu';
2845         $res->{$vmid}->{exists} = 1;
2846     }
2847     return $res;
2848 }
2849
2850 # test if VM uses local resources (to prevent migration)
2851 sub check_local_resources {
2852     my ($conf, $noerr) = @_;
2853
2854     my $loc_res = 0;
2855
2856     $loc_res = 1 if $conf->{hostusb}; # old syntax
2857     $loc_res = 1 if $conf->{hostpci}; # old syntax
2858
2859     $loc_res = 1 if $conf->{ivshmem};
2860
2861     foreach my $k (keys %$conf) {
2862         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2863         # sockets are safe: they will recreated be on the target side post-migrate
2864         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2865         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2866     }
2867
2868     die "VM uses local resources\n" if $loc_res && !$noerr;
2869
2870     return $loc_res;
2871 }
2872
2873 # check if used storages are available on all nodes (use by migrate)
2874 sub check_storage_availability {
2875     my ($storecfg, $conf, $node) = @_;
2876
2877     foreach_drive($conf, sub {
2878         my ($ds, $drive) = @_;
2879
2880         my $volid = $drive->{file};
2881         return if !$volid;
2882
2883         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2884         return if !$sid;
2885
2886         # check if storage is available on both nodes
2887         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2888         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2889    });
2890 }
2891
2892 # list nodes where all VM images are available (used by has_feature API)
2893 sub shared_nodes {
2894     my ($conf, $storecfg) = @_;
2895
2896     my $nodelist = PVE::Cluster::get_nodelist();
2897     my $nodehash = { map { $_ => 1 } @$nodelist };
2898     my $nodename = PVE::INotify::nodename();
2899
2900     foreach_drive($conf, sub {
2901         my ($ds, $drive) = @_;
2902
2903         my $volid = $drive->{file};
2904         return if !$volid;
2905
2906         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2907         if ($storeid) {
2908             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2909             if ($scfg->{disable}) {
2910                 $nodehash = {};
2911             } elsif (my $avail = $scfg->{nodes}) {
2912                 foreach my $node (keys %$nodehash) {
2913                     delete $nodehash->{$node} if !$avail->{$node};
2914                 }
2915             } elsif (!$scfg->{shared}) {
2916                 foreach my $node (keys %$nodehash) {
2917                     delete $nodehash->{$node} if $node ne $nodename
2918                 }
2919             }
2920         }
2921     });
2922
2923     return $nodehash
2924 }
2925
2926 sub check_cmdline {
2927     my ($pidfile, $pid) = @_;
2928
2929     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2930     if (defined($fh)) {
2931         my $line = <$fh>;
2932         $fh->close;
2933         return undef if !$line;
2934         my @param = split(/\0/, $line);
2935
2936         my $cmd = $param[0];
2937         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2938
2939         for (my $i = 0; $i < scalar (@param); $i++) {
2940             my $p = $param[$i];
2941             next if !$p;
2942             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2943                 my $p = $param[$i+1];
2944                 return 1 if $p && ($p eq $pidfile);
2945                 return undef;
2946             }
2947         }
2948     }
2949     return undef;
2950 }
2951
2952 sub check_running {
2953     my ($vmid, $nocheck, $node) = @_;
2954
2955     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2956
2957     die "unable to find configuration file for VM $vmid - no such machine\n"
2958         if !$nocheck && ! -f $filename;
2959
2960     my $pidfile = pidfile_name($vmid);
2961
2962     if (my $fd = IO::File->new("<$pidfile")) {
2963         my $st = stat($fd);
2964         my $line = <$fd>;
2965         close($fd);
2966
2967         my $mtime = $st->mtime;
2968         if ($mtime > time()) {
2969             warn "file '$filename' modified in future\n";
2970         }
2971
2972         if ($line =~ m/^(\d+)$/) {
2973             my $pid = $1;
2974             if (check_cmdline($pidfile, $pid)) {
2975                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2976                     return $pid;
2977                 }
2978             }
2979         }
2980     }
2981
2982     return undef;
2983 }
2984
2985 sub vzlist {
2986
2987     my $vzlist = config_list();
2988
2989     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2990
2991     while (defined(my $de = $fd->read)) {
2992         next if $de !~ m/^(\d+)\.pid$/;
2993         my $vmid = $1;
2994         next if !defined($vzlist->{$vmid});
2995         if (my $pid = check_running($vmid)) {
2996             $vzlist->{$vmid}->{pid} = $pid;
2997         }
2998     }
2999
3000     return $vzlist;
3001 }
3002
3003 sub disksize {
3004     my ($storecfg, $conf) = @_;
3005
3006     my $bootdisk = $conf->{bootdisk};
3007     return undef if !$bootdisk;
3008     return undef if !is_valid_drivename($bootdisk);
3009
3010     return undef if !$conf->{$bootdisk};
3011
3012     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3013     return undef if !defined($drive);
3014
3015     return undef if drive_is_cdrom($drive);
3016
3017     my $volid = $drive->{file};
3018     return undef if !$volid;
3019
3020     return $drive->{size};
3021 }
3022
3023 our $vmstatus_return_properties = {
3024     vmid => get_standard_option('pve-vmid'),
3025     status => {
3026         description => "Qemu process status.",
3027         type => 'string',
3028         enum => ['stopped', 'running'],
3029     },
3030     maxmem => {
3031         description => "Maximum memory in bytes.",
3032         type => 'integer',
3033         optional => 1,
3034         renderer => 'bytes',
3035     },
3036     maxdisk => {
3037         description => "Root disk size in bytes.",
3038         type => 'integer',
3039         optional => 1,
3040         renderer => 'bytes',
3041     },
3042     name => {
3043         description => "VM name.",
3044         type => 'string',
3045         optional => 1,
3046     },
3047     qmpstatus => {
3048         description => "Qemu QMP agent status.",
3049         type => 'string',
3050         optional => 1,
3051     },
3052     pid => {
3053         description => "PID of running qemu process.",
3054         type => 'integer',
3055         optional => 1,
3056     },
3057     uptime => {
3058         description => "Uptime.",
3059         type => 'integer',
3060         optional => 1,
3061         renderer => 'duration',
3062     },
3063     cpus => {
3064         description => "Maximum usable CPUs.",
3065         type => 'number',
3066         optional => 1,
3067     },
3068 };
3069
3070 my $last_proc_pid_stat;
3071
3072 # get VM status information
3073 # This must be fast and should not block ($full == false)
3074 # We only query KVM using QMP if $full == true (this can be slow)
3075 sub vmstatus {
3076     my ($opt_vmid, $full) = @_;
3077
3078     my $res = {};
3079
3080     my $storecfg = PVE::Storage::config();
3081
3082     my $list = vzlist();
3083     my $defaults = load_defaults();
3084
3085     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3086
3087     my $cpucount = $cpuinfo->{cpus} || 1;
3088
3089     foreach my $vmid (keys %$list) {
3090         next if $opt_vmid && ($vmid ne $opt_vmid);
3091
3092         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3093         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3094
3095         my $d = { vmid => $vmid };
3096         $d->{pid} = $list->{$vmid}->{pid};
3097
3098         # fixme: better status?
3099         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3100
3101         my $size = disksize($storecfg, $conf);
3102         if (defined($size)) {
3103             $d->{disk} = 0; # no info available
3104             $d->{maxdisk} = $size;
3105         } else {
3106             $d->{disk} = 0;
3107             $d->{maxdisk} = 0;
3108         }
3109
3110         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3111             * ($conf->{cores} || $defaults->{cores});
3112         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3113         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3114
3115         $d->{name} = $conf->{name} || "VM $vmid";
3116         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3117             : $defaults->{memory}*(1024*1024);
3118
3119         if ($conf->{balloon}) {
3120             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3121             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3122                 : $defaults->{shares};
3123         }
3124
3125         $d->{uptime} = 0;
3126         $d->{cpu} = 0;
3127         $d->{mem} = 0;
3128
3129         $d->{netout} = 0;
3130         $d->{netin} = 0;
3131
3132         $d->{diskread} = 0;
3133         $d->{diskwrite} = 0;
3134
3135         $d->{template} = PVE::QemuConfig->is_template($conf);
3136
3137         $d->{serial} = 1 if conf_has_serial($conf);
3138
3139         $res->{$vmid} = $d;
3140     }
3141
3142     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3143     foreach my $dev (keys %$netdev) {
3144         next if $dev !~ m/^tap([1-9]\d*)i/;
3145         my $vmid = $1;
3146         my $d = $res->{$vmid};
3147         next if !$d;
3148
3149         $d->{netout} += $netdev->{$dev}->{receive};
3150         $d->{netin} += $netdev->{$dev}->{transmit};
3151
3152         if ($full) {
3153             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3154             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3155         }
3156
3157     }
3158
3159     my $ctime = gettimeofday;
3160
3161     foreach my $vmid (keys %$list) {
3162
3163         my $d = $res->{$vmid};
3164         my $pid = $d->{pid};
3165         next if !$pid;
3166
3167         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3168         next if !$pstat; # not running
3169
3170         my $used = $pstat->{utime} + $pstat->{stime};
3171
3172         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3173
3174         if ($pstat->{vsize}) {
3175             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3176         }
3177
3178         my $old = $last_proc_pid_stat->{$pid};
3179         if (!$old) {
3180             $last_proc_pid_stat->{$pid} = {
3181                 time => $ctime,
3182                 used => $used,
3183                 cpu => 0,
3184             };
3185             next;
3186         }
3187
3188         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3189
3190         if ($dtime > 1000) {
3191             my $dutime = $used -  $old->{used};
3192
3193             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3194             $last_proc_pid_stat->{$pid} = {
3195                 time => $ctime,
3196                 used => $used,
3197                 cpu => $d->{cpu},
3198             };
3199         } else {
3200             $d->{cpu} = $old->{cpu};
3201         }
3202     }
3203
3204     return $res if !$full;
3205
3206     my $qmpclient = PVE::QMPClient->new();
3207
3208     my $ballooncb = sub {
3209         my ($vmid, $resp) = @_;
3210
3211         my $info = $resp->{'return'};
3212         return if !$info->{max_mem};
3213
3214         my $d = $res->{$vmid};
3215
3216         # use memory assigned to VM
3217         $d->{maxmem} = $info->{max_mem};
3218         $d->{balloon} = $info->{actual};
3219
3220         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3221             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3222             $d->{freemem} = $info->{free_mem};
3223         }
3224
3225         $d->{ballooninfo} = $info;
3226     };
3227
3228     my $blockstatscb = sub {
3229         my ($vmid, $resp) = @_;
3230         my $data = $resp->{'return'} || [];
3231         my $totalrdbytes = 0;
3232         my $totalwrbytes = 0;
3233
3234         for my $blockstat (@$data) {
3235             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3236             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3237
3238             $blockstat->{device} =~ s/drive-//;
3239             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3240         }
3241         $res->{$vmid}->{diskread} = $totalrdbytes;
3242         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3243     };
3244
3245     my $statuscb = sub {
3246         my ($vmid, $resp) = @_;
3247
3248         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3249         # this fails if ballon driver is not loaded, so this must be
3250         # the last commnand (following command are aborted if this fails).
3251         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3252
3253         my $status = 'unknown';
3254         if (!defined($status = $resp->{'return'}->{status})) {
3255             warn "unable to get VM status\n";
3256             return;
3257         }
3258
3259         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3260     };
3261
3262     foreach my $vmid (keys %$list) {
3263         next if $opt_vmid && ($vmid ne $opt_vmid);
3264         next if !$res->{$vmid}->{pid}; # not running
3265         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3266     }
3267
3268     $qmpclient->queue_execute(undef, 2);
3269
3270     foreach my $vmid (keys %$list) {
3271         next if $opt_vmid && ($vmid ne $opt_vmid);
3272         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3273     }
3274
3275     return $res;
3276 }
3277
3278 sub foreach_drive {
3279     my ($conf, $func, @param) = @_;
3280
3281     foreach my $ds (valid_drive_names()) {
3282         next if !defined($conf->{$ds});
3283
3284         my $drive = parse_drive($ds, $conf->{$ds});
3285         next if !$drive;
3286
3287         &$func($ds, $drive, @param);
3288     }
3289 }
3290
3291 sub foreach_volid {
3292     my ($conf, $func, @param) = @_;
3293
3294     my $volhash = {};
3295
3296     my $test_volid = sub {
3297         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3298
3299         return if !$volid;
3300
3301         $volhash->{$volid}->{cdrom} //= 1;
3302         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3303
3304         $volhash->{$volid}->{replicate} //= 0;
3305         $volhash->{$volid}->{replicate} = 1 if $replicate;
3306
3307         $volhash->{$volid}->{shared} //= 0;
3308         $volhash->{$volid}->{shared} = 1 if $shared;
3309
3310         $volhash->{$volid}->{referenced_in_config} //= 0;
3311         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3312
3313         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3314             if defined($snapname);
3315     };
3316
3317     foreach_drive($conf, sub {
3318         my ($ds, $drive) = @_;
3319         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3320     });
3321
3322     foreach my $snapname (keys %{$conf->{snapshots}}) {
3323         my $snap = $conf->{snapshots}->{$snapname};
3324         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3325         foreach_drive($snap, sub {
3326             my ($ds, $drive) = @_;
3327             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3328         });
3329     }
3330
3331     foreach my $volid (keys %$volhash) {
3332         &$func($volid, $volhash->{$volid}, @param);
3333     }
3334 }
3335
3336 sub conf_has_serial {
3337     my ($conf) = @_;
3338
3339     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3340         if ($conf->{"serial$i"}) {
3341             return 1;
3342         }
3343     }
3344
3345     return 0;
3346 }
3347
3348 sub vga_conf_has_spice {
3349     my ($vga) = @_;
3350
3351     my $vgaconf = parse_vga($vga);
3352     my $vgatype = $vgaconf->{type};
3353     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3354
3355     return $1 || 1;
3356 }
3357
3358 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3359 sub get_host_arch() {
3360     $host_arch = (POSIX::uname())[4] if !$host_arch;
3361     return $host_arch;
3362 }
3363
3364 sub is_native($) {
3365     my ($arch) = @_;
3366     return get_host_arch() eq $arch;
3367 }
3368
3369 my $default_machines = {
3370     x86_64 => 'pc',
3371     aarch64 => 'virt',
3372 };
3373
3374 sub get_basic_machine_info {
3375     my ($conf, $forcemachine) = @_;
3376
3377     my $arch = $conf->{arch} // get_host_arch();
3378     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3379     return ($arch, $machine);
3380 }
3381
3382 sub get_ovmf_files($) {
3383     my ($arch) = @_;
3384
3385     my $ovmf = $OVMF->{$arch}
3386         or die "no OVMF images known for architecture '$arch'\n";
3387
3388     return @$ovmf;
3389 }
3390
3391 my $Arch2Qemu = {
3392     aarch64 => '/usr/bin/qemu-system-aarch64',
3393     x86_64 => '/usr/bin/qemu-system-x86_64',
3394 };
3395 sub get_command_for_arch($) {
3396     my ($arch) = @_;
3397     return '/usr/bin/kvm' if is_native($arch);
3398
3399     my $cmd = $Arch2Qemu->{$arch}
3400         or die "don't know how to emulate architecture '$arch'\n";
3401     return $cmd;
3402 }
3403
3404 sub get_cpu_options {
3405     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3406
3407     my $cpuFlags = [];
3408     my $ostype = $conf->{ostype};
3409
3410     my $cpu = $kvm ? "kvm64" : "qemu64";
3411     if ($arch eq 'aarch64') {
3412         $cpu = 'cortex-a57';
3413     }
3414     my $hv_vendor_id;
3415     if (my $cputype = $conf->{cpu}) {
3416         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3417             or die "Cannot parse cpu description: $cputype\n";
3418         $cpu = $cpuconf->{cputype};
3419         $kvm_off = 1 if $cpuconf->{hidden};
3420         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3421
3422         if (defined(my $flags = $cpuconf->{flags})) {
3423             push @$cpuFlags, split(";", $flags);
3424         }
3425     }
3426
3427     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3428
3429     push @$cpuFlags , '-x2apic'
3430         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3431
3432     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3433
3434     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3435
3436     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3437
3438         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3439         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3440     }
3441
3442     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3443
3444     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3445
3446     push @$cpuFlags, 'kvm=off' if $kvm_off;
3447
3448     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3449         push @$cpuFlags, "vendor=${cpu_vendor}"
3450             if $cpu_vendor ne 'default';
3451     } elsif ($arch ne 'aarch64') {
3452         die "internal error"; # should not happen
3453     }
3454
3455     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3456
3457     return ('-cpu', $cpu);
3458 }
3459
3460 sub config_to_command {
3461     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3462
3463     my $cmd = [];
3464     my $globalFlags = [];
3465     my $machineFlags = [];
3466     my $rtcFlags = [];
3467     my $devices = [];
3468     my $pciaddr = '';
3469     my $bridges = {};
3470     my $kvmver = kvm_user_version();
3471     my $vernum = 0; # unknown
3472     my $ostype = $conf->{ostype};
3473     my $winversion = windows_version($ostype);
3474     my $kvm = $conf->{kvm};
3475
3476     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3477     $kvm //= 1 if is_native($arch);
3478
3479     if ($kvm) {
3480         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3481             if !defined kvm_version();
3482     }
3483
3484     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3485         $vernum = $1*1000000+$2*1000;
3486     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3487         $vernum = $1*1000000+$2*1000+$3;
3488     }
3489
3490     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3491
3492     my $have_ovz = -f '/proc/vz/vestat';
3493
3494     my $q35 = machine_type_is_q35($conf);
3495     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3496     my $use_old_bios_files = undef;
3497     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3498
3499     my $cpuunits = defined($conf->{cpuunits}) ?
3500             $conf->{cpuunits} : $defaults->{cpuunits};
3501
3502     push @$cmd, get_command_for_arch($arch);
3503
3504     push @$cmd, '-id', $vmid;
3505
3506     my $vmname = $conf->{name} || "vm$vmid";
3507
3508     push @$cmd, '-name', $vmname;
3509
3510     my $use_virtio = 0;
3511
3512     my $qmpsocket = qmp_socket($vmid);
3513     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3514     push @$cmd, '-mon', "chardev=qmp,mode=control";
3515
3516     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3517         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3518         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3519     }
3520
3521     push @$cmd, '-pidfile' , pidfile_name($vmid);
3522
3523     push @$cmd, '-daemonize';
3524
3525     if ($conf->{smbios1}) {
3526         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3527     }
3528
3529     if ($conf->{vmgenid}) {
3530         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3531     }
3532
3533     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3534     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3535         die "uefi base image not found\n" if ! -f $ovmf_code;
3536
3537         my $path;
3538         my $format;
3539         if (my $efidisk = $conf->{efidisk0}) {
3540             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3541             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3542             $format = $d->{format};
3543             if ($storeid) {
3544                 $path = PVE::Storage::path($storecfg, $d->{file});
3545                 if (!defined($format)) {
3546                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3547                     $format = qemu_img_format($scfg, $volname);
3548                 }
3549             } else {
3550                 $path = $d->{file};
3551                 die "efidisk format must be specified\n"
3552                     if !defined($format);
3553             }
3554         } else {
3555             warn "no efidisk configured! Using temporary efivars disk.\n";
3556             $path = "/tmp/$vmid-ovmf.fd";
3557             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3558             $format = 'raw';
3559         }
3560
3561         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3562         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3563     }
3564
3565
3566     # add usb controllers
3567     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3568     push @$devices, @usbcontrollers if @usbcontrollers;
3569     my $vga = parse_vga($conf->{vga});
3570
3571     my $qxlnum = vga_conf_has_spice($conf->{vga});
3572     $vga->{type} = 'qxl' if $qxlnum;
3573
3574     if (!$vga->{type}) {
3575         if ($arch eq 'aarch64') {
3576             $vga->{type} = 'virtio';
3577         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3578             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3579         } else {
3580             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3581         }
3582     }
3583
3584     # enable absolute mouse coordinates (needed by vnc)
3585     my $tablet;
3586     if (defined($conf->{tablet})) {
3587         $tablet = $conf->{tablet};
3588     } else {
3589         $tablet = $defaults->{tablet};
3590         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3591         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3592     }
3593
3594     if ($tablet) {
3595         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3596         my $kbd = print_keyboarddevice_full($conf, $arch);
3597         push @$devices, '-device', $kbd if defined($kbd);
3598     }
3599
3600     my $kvm_off = 0;
3601     my $gpu_passthrough;
3602
3603     # host pci devices
3604     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3605         my $d = parse_hostpci($conf->{"hostpci$i"});
3606         next if !$d;
3607
3608         my $pcie = $d->{pcie};
3609         if ($pcie) {
3610             die "q35 machine model is not enabled" if !$q35;
3611             # win7 wants to have the pcie devices directly on the pcie bus
3612             # instead of in the root port
3613             if ($winversion == 7) {
3614                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3615             } else {
3616                 $pciaddr = print_pcie_addr("hostpci$i");
3617             }
3618         } else {
3619             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3620         }
3621
3622         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3623         my $romfile = $d->{romfile};
3624
3625         my $xvga = '';
3626         if ($d->{'x-vga'}) {
3627             $xvga = ',x-vga=on';
3628             $kvm_off = 1;
3629             $vga->{type} = 'none' if !defined($conf->{vga});
3630             $gpu_passthrough = 1;
3631
3632             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3633                 $xvga = "";
3634             }
3635         }
3636         my $pcidevices = $d->{pciid};
3637         my $multifunction = 1 if @$pcidevices > 1;
3638         my $sysfspath;
3639         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3640             my $id = $pcidevices->[0]->{id};
3641             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3642             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3643         } elsif ($d->{mdev}) {
3644             warn "ignoring mediated device with multifunction device\n";
3645         }
3646
3647         my $j=0;
3648         foreach my $pcidevice (@$pcidevices) {
3649
3650             my $id = "hostpci$i";
3651             $id .= ".$j" if $multifunction;
3652             my $addr = $pciaddr;
3653             $addr .= ".$j" if $multifunction;
3654             my $devicestr = "vfio-pci";
3655             if ($sysfspath) {
3656                 $devicestr .= ",sysfsdev=$sysfspath";
3657             } else {
3658                 $devicestr .= ",host=$pcidevice->{id}";
3659             }
3660             $devicestr .= ",id=$id$addr";
3661
3662             if($j == 0){
3663                 $devicestr .= "$rombar$xvga";
3664                 $devicestr .= ",multifunction=on" if $multifunction;
3665                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3666             }
3667
3668             push @$devices, '-device', $devicestr;
3669             $j++;
3670         }
3671     }
3672
3673     # usb devices
3674     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3675     push @$devices, @usbdevices if @usbdevices;
3676     # serial devices
3677     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3678         if (my $path = $conf->{"serial$i"}) {
3679             if ($path eq 'socket') {
3680                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3681                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3682                 # On aarch64, serial0 is the UART device. Qemu only allows
3683                 # connecting UART devices via the '-serial' command line, as
3684                 # the device has a fixed slot on the hardware...
3685                 if ($arch eq 'aarch64' && $i == 0) {
3686                     push @$devices, '-serial', "chardev:serial$i";
3687                 } else {
3688                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3689                 }
3690             } else {
3691                 die "no such serial device\n" if ! -c $path;
3692                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3693                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3694             }
3695         }
3696     }
3697
3698     # parallel devices
3699     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3700         if (my $path = $conf->{"parallel$i"}) {
3701             die "no such parallel device\n" if ! -c $path;
3702             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3703             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3704             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3705         }
3706     }
3707
3708
3709     my $sockets = 1;
3710     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3711     $sockets = $conf->{sockets} if  $conf->{sockets};
3712
3713     my $cores = $conf->{cores} || 1;
3714
3715     my $maxcpus = $sockets * $cores;
3716
3717     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3718
3719     my $allowed_vcpus = $cpuinfo->{cpus};
3720
3721     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3722         if ($allowed_vcpus < $maxcpus);
3723
3724     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3725
3726         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3727         for (my $i = 2; $i <= $vcpus; $i++)  {
3728             my $cpustr = print_cpu_device($conf,$i);
3729             push @$cmd, '-device', $cpustr;
3730         }
3731
3732     } else {
3733
3734         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3735     }
3736     push @$cmd, '-nodefaults';
3737
3738     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3739
3740     my $bootindex_hash = {};
3741     my $i = 1;
3742     foreach my $o (split(//, $bootorder)) {
3743         $bootindex_hash->{$o} = $i*100;
3744         $i++;
3745     }
3746
3747     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3748
3749     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3750
3751     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3752
3753     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3754         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3755         my $socket = vnc_socket($vmid);
3756         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3757     } else {
3758         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3759         push @$cmd, '-nographic';
3760     }
3761
3762     # time drift fix
3763     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3764
3765     my $useLocaltime = $conf->{localtime};
3766
3767     if ($winversion >= 5) { # windows
3768         $useLocaltime = 1 if !defined($conf->{localtime});
3769
3770         # use time drift fix when acpi is enabled
3771         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3772             $tdf = 1 if !defined($conf->{tdf});
3773         }
3774     }
3775
3776     if ($winversion >= 6) {
3777         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3778         push @$cmd, '-no-hpet';
3779     }
3780
3781     push @$rtcFlags, 'driftfix=slew' if $tdf;
3782
3783     if (!$kvm) {
3784         push @$machineFlags, 'accel=tcg';
3785     }
3786
3787     if ($machine_type) {
3788         push @$machineFlags, "type=${machine_type}";
3789     }
3790
3791     if ($conf->{startdate}) {
3792         push @$rtcFlags, "base=$conf->{startdate}";
3793     } elsif ($useLocaltime) {
3794         push @$rtcFlags, 'base=localtime';
3795     }
3796
3797     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3798
3799     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3800
3801     push @$cmd, '-S' if $conf->{freeze};
3802
3803     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3804
3805     # enable sound
3806     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3807     #push @$cmd, '-soundhw', 'es1370';
3808     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3809
3810     if (parse_guest_agent($conf)->{enabled}) {
3811         my $qgasocket = qmp_socket($vmid, 1);
3812         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3813         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3814         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3815         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3816     }
3817
3818     my $spice_port;
3819
3820     if ($qxlnum) {
3821         if ($qxlnum > 1) {
3822             if ($winversion){
3823                 for(my $i = 1; $i < $qxlnum; $i++){
3824                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3825                 }
3826             } else {
3827                 # assume other OS works like Linux
3828                 my ($ram, $vram) = ("134217728", "67108864");
3829                 if ($vga->{memory}) {
3830                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3831                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3832                 }
3833                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3834                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3835             }
3836         }
3837
3838         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3839
3840         my $nodename = PVE::INotify::nodename();
3841         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3842         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3843         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3844         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3845         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3846
3847         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3848
3849         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3850         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3851         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3852     }
3853
3854     # enable balloon by default, unless explicitly disabled
3855     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3856         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3857         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3858     }
3859
3860     if ($conf->{watchdog}) {
3861         my $wdopts = parse_watchdog($conf->{watchdog});
3862         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3863         my $watchdog = $wdopts->{model} || 'i6300esb';
3864         push @$devices, '-device', "$watchdog$pciaddr";
3865         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3866     }
3867
3868     my $vollist = [];
3869     my $scsicontroller = {};
3870     my $ahcicontroller = {};
3871     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3872
3873     # Add iscsi initiator name if available
3874     if (my $initiator = get_initiator_name()) {
3875         push @$devices, '-iscsi', "initiator-name=$initiator";
3876     }
3877
3878     foreach_drive($conf, sub {
3879         my ($ds, $drive) = @_;
3880
3881         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3882             push @$vollist, $drive->{file};
3883         }
3884
3885         # ignore efidisk here, already added in bios/fw handling code above
3886         return if $drive->{interface} eq 'efidisk';
3887
3888         $use_virtio = 1 if $ds =~ m/^virtio/;
3889
3890         if (drive_is_cdrom ($drive)) {
3891             if ($bootindex_hash->{d}) {
3892                 $drive->{bootindex} = $bootindex_hash->{d};
3893                 $bootindex_hash->{d} += 1;
3894             }
3895         } else {
3896             if ($bootindex_hash->{c}) {
3897                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3898                 $bootindex_hash->{c} += 1;
3899             }
3900         }
3901
3902         if($drive->{interface} eq 'virtio'){
3903            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3904         }
3905
3906         if ($drive->{interface} eq 'scsi') {
3907
3908             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3909
3910             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3911             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3912
3913             my $iothread = '';
3914             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3915                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3916                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3917             } elsif ($drive->{iothread}) {
3918                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3919             }
3920
3921             my $queues = '';
3922             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3923                 $queues = ",num_queues=$drive->{queues}";
3924             }
3925
3926             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3927             $scsicontroller->{$controller}=1;
3928         }
3929
3930         if ($drive->{interface} eq 'sata') {
3931            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3932            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3933            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3934            $ahcicontroller->{$controller}=1;
3935         }
3936
3937         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3938         push @$devices, '-drive',$drive_cmd;
3939         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3940     });
3941
3942     for (my $i = 0; $i < $MAX_NETS; $i++) {
3943          next if !$conf->{"net$i"};
3944          my $d = parse_net($conf->{"net$i"});
3945          next if !$d;
3946
3947          $use_virtio = 1 if $d->{model} eq 'virtio';
3948
3949          if ($bootindex_hash->{n}) {
3950             $d->{bootindex} = $bootindex_hash->{n};
3951             $bootindex_hash->{n} += 1;
3952          }
3953
3954          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3955          push @$devices, '-netdev', $netdevfull;
3956
3957          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3958          push @$devices, '-device', $netdevicefull;
3959     }
3960
3961     if ($conf->{ivshmem}) {
3962         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3963
3964         my $bus;
3965         if ($q35) {
3966             $bus = print_pcie_addr("ivshmem");
3967         } else {
3968             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3969         }
3970
3971         my $ivshmem_name = $ivshmem->{name} // $vmid;
3972         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3973
3974         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3975         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3976     }
3977
3978     if (!$q35) {
3979         # add pci bridges
3980         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3981            $bridges->{1} = 1;
3982            $bridges->{2} = 1;
3983         }
3984
3985         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3986
3987         while (my ($k, $v) = each %$bridges) {
3988             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3989             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3990         }
3991     }
3992
3993     push @$cmd, @$devices;
3994     push @$cmd, '-rtc', join(',', @$rtcFlags)
3995         if scalar(@$rtcFlags);
3996     push @$cmd, '-machine', join(',', @$machineFlags)
3997         if scalar(@$machineFlags);
3998     push @$cmd, '-global', join(',', @$globalFlags)
3999         if scalar(@$globalFlags);
4000
4001     if (my $vmstate = $conf->{vmstate}) {
4002         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4003         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4004         push @$cmd, '-loadstate', $statepath;
4005     }
4006
4007     # add custom args
4008     if ($conf->{args}) {
4009         my $aa = PVE::Tools::split_args($conf->{args});
4010         push @$cmd, @$aa;
4011     }
4012
4013     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4014 }
4015
4016 sub vnc_socket {
4017     my ($vmid) = @_;
4018     return "${var_run_tmpdir}/$vmid.vnc";
4019 }
4020
4021 sub spice_port {
4022     my ($vmid) = @_;
4023
4024     my $res = vm_mon_cmd($vmid, 'query-spice');
4025
4026     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4027 }
4028
4029 sub qmp_socket {
4030     my ($vmid, $qga, $name) = @_;
4031     my $sockettype = $qga ? 'qga' : 'qmp';
4032     my $ext = $name ? '-'.$name : '';
4033     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4034 }
4035
4036 sub pidfile_name {
4037     my ($vmid) = @_;
4038     return "${var_run_tmpdir}/$vmid.pid";
4039 }
4040
4041 sub vm_devices_list {
4042     my ($vmid) = @_;
4043
4044     my $res = vm_mon_cmd($vmid, 'query-pci');
4045     my $devices_to_check = [];
4046     my $devices = {};
4047     foreach my $pcibus (@$res) {
4048         push @$devices_to_check, @{$pcibus->{devices}},
4049     }
4050
4051     while (@$devices_to_check) {
4052         my $to_check = [];
4053         for my $d (@$devices_to_check) {
4054             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4055             next if !$d->{'pci_bridge'};
4056
4057             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4058             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4059         }
4060         $devices_to_check = $to_check;
4061     }
4062
4063     my $resblock = vm_mon_cmd($vmid, 'query-block');
4064     foreach my $block (@$resblock) {
4065         if($block->{device} =~ m/^drive-(\S+)/){
4066                 $devices->{$1} = 1;
4067         }
4068     }
4069
4070     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4071     foreach my $mice (@$resmice) {
4072         if ($mice->{name} eq 'QEMU HID Tablet') {
4073             $devices->{tablet} = 1;
4074             last;
4075         }
4076     }
4077
4078     # for usb devices there is no query-usb
4079     # but we can iterate over the entries in
4080     # qom-list path=/machine/peripheral
4081     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4082     foreach my $per (@$resperipheral) {
4083         if ($per->{name} =~ m/^usb\d+$/) {
4084             $devices->{$per->{name}} = 1;
4085         }
4086     }
4087
4088     return $devices;
4089 }
4090
4091 sub vm_deviceplug {
4092     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4093
4094     my $q35 = machine_type_is_q35($conf);
4095
4096     my $devices_list = vm_devices_list($vmid);
4097     return 1 if defined($devices_list->{$deviceid});
4098
4099     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4100
4101     if ($deviceid eq 'tablet') {
4102
4103         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4104
4105     } elsif ($deviceid eq 'keyboard') {
4106
4107         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4108
4109     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4110
4111         die "usb hotplug currently not reliable\n";
4112         # since we can't reliably hot unplug all added usb devices
4113         # and usb passthrough disables live migration
4114         # we disable usb hotplugging for now
4115         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4116
4117     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4118
4119         qemu_iothread_add($vmid, $deviceid, $device);
4120
4121         qemu_driveadd($storecfg, $vmid, $device);
4122         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4123
4124         qemu_deviceadd($vmid, $devicefull);
4125         eval { qemu_deviceaddverify($vmid, $deviceid); };
4126         if (my $err = $@) {
4127             eval { qemu_drivedel($vmid, $deviceid); };
4128             warn $@ if $@;
4129             die $err;
4130         }
4131
4132     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4133
4134
4135         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4136         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4137         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4138
4139         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4140
4141         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4142             qemu_iothread_add($vmid, $deviceid, $device);
4143             $devicefull .= ",iothread=iothread-$deviceid";
4144         }
4145
4146         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4147             $devicefull .= ",num_queues=$device->{queues}";
4148         }
4149
4150         qemu_deviceadd($vmid, $devicefull);
4151         qemu_deviceaddverify($vmid, $deviceid);
4152
4153     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4154
4155         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4156         qemu_driveadd($storecfg, $vmid, $device);
4157
4158         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4159         eval { qemu_deviceadd($vmid, $devicefull); };
4160         if (my $err = $@) {
4161             eval { qemu_drivedel($vmid, $deviceid); };
4162             warn $@ if $@;
4163             die $err;
4164         }
4165
4166     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4167
4168         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4169
4170         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4171         my $use_old_bios_files = undef;
4172         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4173
4174         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4175         qemu_deviceadd($vmid, $netdevicefull);
4176         eval {
4177             qemu_deviceaddverify($vmid, $deviceid);
4178             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4179         };
4180         if (my $err = $@) {
4181             eval { qemu_netdevdel($vmid, $deviceid); };
4182             warn $@ if $@;
4183             die $err;
4184         }
4185
4186     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4187
4188         my $bridgeid = $2;
4189         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4190         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4191
4192         qemu_deviceadd($vmid, $devicefull);
4193         qemu_deviceaddverify($vmid, $deviceid);
4194
4195     } else {
4196         die "can't hotplug device '$deviceid'\n";
4197     }
4198
4199     return 1;
4200 }
4201
4202 # fixme: this should raise exceptions on error!
4203 sub vm_deviceunplug {
4204     my ($vmid, $conf, $deviceid) = @_;
4205
4206     my $devices_list = vm_devices_list($vmid);
4207     return 1 if !defined($devices_list->{$deviceid});
4208
4209     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4210
4211     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4212
4213         qemu_devicedel($vmid, $deviceid);
4214
4215     } elsif ($deviceid =~ m/^usb\d+$/) {
4216
4217         die "usb hotplug currently not reliable\n";
4218         # when unplugging usb devices this way,
4219         # there may be remaining usb controllers/hubs
4220         # so we disable it for now
4221         qemu_devicedel($vmid, $deviceid);
4222         qemu_devicedelverify($vmid, $deviceid);
4223
4224     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4225
4226         qemu_devicedel($vmid, $deviceid);
4227         qemu_devicedelverify($vmid, $deviceid);
4228         qemu_drivedel($vmid, $deviceid);
4229         qemu_iothread_del($conf, $vmid, $deviceid);
4230
4231     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4232
4233         qemu_devicedel($vmid, $deviceid);
4234         qemu_devicedelverify($vmid, $deviceid);
4235         qemu_iothread_del($conf, $vmid, $deviceid);
4236
4237     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4238
4239         qemu_devicedel($vmid, $deviceid);
4240         qemu_drivedel($vmid, $deviceid);
4241         qemu_deletescsihw($conf, $vmid, $deviceid);
4242
4243     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4244
4245         qemu_devicedel($vmid, $deviceid);
4246         qemu_devicedelverify($vmid, $deviceid);
4247         qemu_netdevdel($vmid, $deviceid);
4248
4249     } else {
4250         die "can't unplug device '$deviceid'\n";
4251     }
4252
4253     return 1;
4254 }
4255
4256 sub qemu_deviceadd {
4257     my ($vmid, $devicefull) = @_;
4258
4259     $devicefull = "driver=".$devicefull;
4260     my %options =  split(/[=,]/, $devicefull);
4261
4262     vm_mon_cmd($vmid, "device_add" , %options);
4263 }
4264
4265 sub qemu_devicedel {
4266     my ($vmid, $deviceid) = @_;
4267
4268     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4269 }
4270
4271 sub qemu_iothread_add {
4272     my($vmid, $deviceid, $device) = @_;
4273
4274     if ($device->{iothread}) {
4275         my $iothreads = vm_iothreads_list($vmid);
4276         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4277     }
4278 }
4279
4280 sub qemu_iothread_del {
4281     my($conf, $vmid, $deviceid) = @_;
4282
4283     my $confid = $deviceid;
4284     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4285         $confid = 'scsi' . $1;
4286     }
4287     my $device = parse_drive($confid, $conf->{$confid});
4288     if ($device->{iothread}) {
4289         my $iothreads = vm_iothreads_list($vmid);
4290         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4291     }
4292 }
4293
4294 sub qemu_objectadd {
4295     my($vmid, $objectid, $qomtype) = @_;
4296
4297     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4298
4299     return 1;
4300 }
4301
4302 sub qemu_objectdel {
4303     my($vmid, $objectid) = @_;
4304
4305     vm_mon_cmd($vmid, "object-del", id => $objectid);
4306
4307     return 1;
4308 }
4309
4310 sub qemu_driveadd {
4311     my ($storecfg, $vmid, $device) = @_;
4312
4313     my $drive = print_drive_full($storecfg, $vmid, $device);
4314     $drive =~ s/\\/\\\\/g;
4315     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4316
4317     # If the command succeeds qemu prints: "OK"
4318     return 1 if $ret =~ m/OK/s;
4319
4320     die "adding drive failed: $ret\n";
4321 }
4322
4323 sub qemu_drivedel {
4324     my($vmid, $deviceid) = @_;
4325
4326     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4327     $ret =~ s/^\s+//;
4328
4329     return 1 if $ret eq "";
4330
4331     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4332     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4333
4334     die "deleting drive $deviceid failed : $ret\n";
4335 }
4336
4337 sub qemu_deviceaddverify {
4338     my ($vmid, $deviceid) = @_;
4339
4340     for (my $i = 0; $i <= 5; $i++) {
4341          my $devices_list = vm_devices_list($vmid);
4342          return 1 if defined($devices_list->{$deviceid});
4343          sleep 1;
4344     }
4345
4346     die "error on hotplug device '$deviceid'\n";
4347 }
4348
4349
4350 sub qemu_devicedelverify {
4351     my ($vmid, $deviceid) = @_;
4352
4353     # need to verify that the device is correctly removed as device_del
4354     # is async and empty return is not reliable
4355
4356     for (my $i = 0; $i <= 5; $i++) {
4357          my $devices_list = vm_devices_list($vmid);
4358          return 1 if !defined($devices_list->{$deviceid});
4359          sleep 1;
4360     }
4361
4362     die "error on hot-unplugging device '$deviceid'\n";
4363 }
4364
4365 sub qemu_findorcreatescsihw {
4366     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4367
4368     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4369
4370     my $scsihwid="$controller_prefix$controller";
4371     my $devices_list = vm_devices_list($vmid);
4372
4373     if(!defined($devices_list->{$scsihwid})) {
4374         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4375     }
4376
4377     return 1;
4378 }
4379
4380 sub qemu_deletescsihw {
4381     my ($conf, $vmid, $opt) = @_;
4382
4383     my $device = parse_drive($opt, $conf->{$opt});
4384
4385     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4386         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4387         return 1;
4388     }
4389
4390     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4391
4392     my $devices_list = vm_devices_list($vmid);
4393     foreach my $opt (keys %{$devices_list}) {
4394         if (PVE::QemuServer::is_valid_drivename($opt)) {
4395             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4396             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4397                 return 1;
4398             }
4399         }
4400     }
4401
4402     my $scsihwid="scsihw$controller";
4403
4404     vm_deviceunplug($vmid, $conf, $scsihwid);
4405
4406     return 1;
4407 }
4408
4409 sub qemu_add_pci_bridge {
4410     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4411
4412     my $bridges = {};
4413
4414     my $bridgeid;
4415
4416     print_pci_addr($device, $bridges, $arch, $machine_type);
4417
4418     while (my ($k, $v) = each %$bridges) {
4419         $bridgeid = $k;
4420     }
4421     return 1 if !defined($bridgeid) || $bridgeid < 1;
4422
4423     my $bridge = "pci.$bridgeid";
4424     my $devices_list = vm_devices_list($vmid);
4425
4426     if (!defined($devices_list->{$bridge})) {
4427         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4428     }
4429
4430     return 1;
4431 }
4432
4433 sub qemu_set_link_status {
4434     my ($vmid, $device, $up) = @_;
4435
4436     vm_mon_cmd($vmid, "set_link", name => $device,
4437                up => $up ? JSON::true : JSON::false);
4438 }
4439
4440 sub qemu_netdevadd {
4441     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4442
4443     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4444     my %options =  split(/[=,]/, $netdev);
4445
4446     vm_mon_cmd($vmid, "netdev_add",  %options);
4447     return 1;
4448 }
4449
4450 sub qemu_netdevdel {
4451     my ($vmid, $deviceid) = @_;
4452
4453     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4454 }
4455
4456 sub qemu_usb_hotplug {
4457     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4458
4459     return if !$device;
4460
4461     # remove the old one first
4462     vm_deviceunplug($vmid, $conf, $deviceid);
4463
4464     # check if xhci controller is necessary and available
4465     if ($device->{usb3}) {
4466
4467         my $devicelist = vm_devices_list($vmid);
4468
4469         if (!$devicelist->{xhci}) {
4470             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4471             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4472         }
4473     }
4474     my $d = parse_usb_device($device->{host});
4475     $d->{usb3} = $device->{usb3};
4476
4477     # add the new one
4478     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4479 }
4480
4481 sub qemu_cpu_hotplug {
4482     my ($vmid, $conf, $vcpus) = @_;
4483
4484     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4485
4486     my $sockets = 1;
4487     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4488     $sockets = $conf->{sockets} if  $conf->{sockets};
4489     my $cores = $conf->{cores} || 1;
4490     my $maxcpus = $sockets * $cores;
4491
4492     $vcpus = $maxcpus if !$vcpus;
4493
4494     die "you can't add more vcpus than maxcpus\n"
4495         if $vcpus > $maxcpus;
4496
4497     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4498
4499     if ($vcpus < $currentvcpus) {
4500
4501         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4502
4503             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4504                 qemu_devicedel($vmid, "cpu$i");
4505                 my $retry = 0;
4506                 my $currentrunningvcpus = undef;
4507                 while (1) {
4508                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4509                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4510                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4511                     $retry++;
4512                     sleep 1;
4513                 }
4514                 #update conf after each succesfull cpu unplug
4515                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4516                 PVE::QemuConfig->write_config($vmid, $conf);
4517             }
4518         } else {
4519             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4520         }
4521
4522         return;
4523     }
4524
4525     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4526     die "vcpus in running vm does not match its configuration\n"
4527         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4528
4529     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4530
4531         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4532             my $cpustr = print_cpu_device($conf, $i);
4533             qemu_deviceadd($vmid, $cpustr);
4534
4535             my $retry = 0;
4536             my $currentrunningvcpus = undef;
4537             while (1) {
4538                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4539                 last if scalar(@{$currentrunningvcpus}) == $i;
4540                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4541                 sleep 1;
4542                 $retry++;
4543             }
4544             #update conf after each succesfull cpu hotplug
4545             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4546             PVE::QemuConfig->write_config($vmid, $conf);
4547         }
4548     } else {
4549
4550         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4551             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4552         }
4553     }
4554 }
4555
4556 sub qemu_block_set_io_throttle {
4557     my ($vmid, $deviceid,
4558         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4559         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4560         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4561         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4562
4563     return if !check_running($vmid) ;
4564
4565     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4566         bps => int($bps),
4567         bps_rd => int($bps_rd),
4568         bps_wr => int($bps_wr),
4569         iops => int($iops),
4570         iops_rd => int($iops_rd),
4571         iops_wr => int($iops_wr),
4572         bps_max => int($bps_max),
4573         bps_rd_max => int($bps_rd_max),
4574         bps_wr_max => int($bps_wr_max),
4575         iops_max => int($iops_max),
4576         iops_rd_max => int($iops_rd_max),
4577         iops_wr_max => int($iops_wr_max),
4578         bps_max_length => int($bps_max_length),
4579         bps_rd_max_length => int($bps_rd_max_length),
4580         bps_wr_max_length => int($bps_wr_max_length),
4581         iops_max_length => int($iops_max_length),
4582         iops_rd_max_length => int($iops_rd_max_length),
4583         iops_wr_max_length => int($iops_wr_max_length),
4584     );
4585
4586 }
4587
4588 # old code, only used to shutdown old VM after update
4589 sub __read_avail {
4590     my ($fh, $timeout) = @_;
4591
4592     my $sel = new IO::Select;
4593     $sel->add($fh);
4594
4595     my $res = '';
4596     my $buf;
4597
4598     my @ready;
4599     while (scalar (@ready = $sel->can_read($timeout))) {
4600         my $count;
4601         if ($count = $fh->sysread($buf, 8192)) {
4602             if ($buf =~ /^(.*)\(qemu\) $/s) {
4603                 $res .= $1;
4604                 last;
4605             } else {
4606                 $res .= $buf;
4607             }
4608         } else {
4609             if (!defined($count)) {
4610                 die "$!\n";
4611             }
4612             last;
4613         }
4614     }
4615
4616     die "monitor read timeout\n" if !scalar(@ready);
4617
4618     return $res;
4619 }
4620
4621 sub qemu_block_resize {
4622     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4623
4624     my $running = check_running($vmid);
4625
4626     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4627
4628     return if !$running;
4629
4630     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4631
4632 }
4633
4634 sub qemu_volume_snapshot {
4635     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4636
4637     my $running = check_running($vmid);
4638
4639     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4640         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4641     } else {
4642         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4643     }
4644 }
4645
4646 sub qemu_volume_snapshot_delete {
4647     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4648
4649     my $running = check_running($vmid);
4650
4651     if($running) {
4652
4653         $running = undef;
4654         my $conf = PVE::QemuConfig->load_config($vmid);
4655         foreach_drive($conf, sub {
4656             my ($ds, $drive) = @_;
4657             $running = 1 if $drive->{file} eq $volid;
4658         });
4659     }
4660
4661     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4662         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4663     } else {
4664         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4665     }
4666 }
4667
4668 sub set_migration_caps {
4669     my ($vmid) = @_;
4670
4671     my $cap_ref = [];
4672
4673     my $enabled_cap = {
4674         "auto-converge" => 1,
4675         "xbzrle" => 1,
4676         "x-rdma-pin-all" => 0,
4677         "zero-blocks" => 0,
4678         "compress" => 0
4679     };
4680
4681     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4682
4683     for my $supported_capability (@$supported_capabilities) {
4684         push @$cap_ref, {
4685             capability => $supported_capability->{capability},
4686             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4687         };
4688     }
4689
4690     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4691 }
4692
4693 my $fast_plug_option = {
4694     'lock' => 1,
4695     'name' => 1,
4696     'onboot' => 1,
4697     'shares' => 1,
4698     'startup' => 1,
4699     'description' => 1,
4700     'protection' => 1,
4701     'vmstatestorage' => 1,
4702     'hookscript' => 1,
4703 };
4704
4705 # hotplug changes in [PENDING]
4706 # $selection hash can be used to only apply specified options, for
4707 # example: { cores => 1 } (only apply changed 'cores')
4708 # $errors ref is used to return error messages
4709 sub vmconfig_hotplug_pending {
4710     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4711
4712     my $defaults = load_defaults();
4713     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4714
4715     # commit values which do not have any impact on running VM first
4716     # Note: those option cannot raise errors, we we do not care about
4717     # $selection and always apply them.
4718
4719     my $add_error = sub {
4720         my ($opt, $msg) = @_;
4721         $errors->{$opt} = "hotplug problem - $msg";
4722     };
4723
4724     my $changes = 0;
4725     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4726         if ($fast_plug_option->{$opt}) {
4727             $conf->{$opt} = $conf->{pending}->{$opt};
4728             delete $conf->{pending}->{$opt};
4729             $changes = 1;
4730         }
4731     }
4732
4733     if ($changes) {
4734         PVE::QemuConfig->write_config($vmid, $conf);
4735         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4736     }
4737
4738     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4739
4740     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4741     while (my ($opt, $force) = each %$pending_delete_hash) {
4742         next if $selection && !$selection->{$opt};
4743         eval {
4744             if ($opt eq 'hotplug') {
4745                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4746             } elsif ($opt eq 'tablet') {
4747                 die "skip\n" if !$hotplug_features->{usb};
4748                 if ($defaults->{tablet}) {
4749                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4750                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4751                         if $arch eq 'aarch64';
4752                 } else {
4753                     vm_deviceunplug($vmid, $conf, 'tablet');
4754                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4755                 }
4756             } elsif ($opt =~ m/^usb\d+/) {
4757                 die "skip\n";
4758                 # since we cannot reliably hot unplug usb devices
4759                 # we are disabling it
4760                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4761                 vm_deviceunplug($vmid, $conf, $opt);
4762             } elsif ($opt eq 'vcpus') {
4763                 die "skip\n" if !$hotplug_features->{cpu};
4764                 qemu_cpu_hotplug($vmid, $conf, undef);
4765             } elsif ($opt eq 'balloon') {
4766                 # enable balloon device is not hotpluggable
4767                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4768                 # here we reset the ballooning value to memory
4769                 my $balloon = $conf->{memory} || $defaults->{memory};
4770                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4771             } elsif ($fast_plug_option->{$opt}) {
4772                 # do nothing
4773             } elsif ($opt =~ m/^net(\d+)$/) {
4774                 die "skip\n" if !$hotplug_features->{network};
4775                 vm_deviceunplug($vmid, $conf, $opt);
4776             } elsif (is_valid_drivename($opt)) {
4777                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4778                 vm_deviceunplug($vmid, $conf, $opt);
4779                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4780             } elsif ($opt =~ m/^memory$/) {
4781                 die "skip\n" if !$hotplug_features->{memory};
4782                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4783             } elsif ($opt eq 'cpuunits') {
4784                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4785             } elsif ($opt eq 'cpulimit') {
4786                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4787             } else {
4788                 die "skip\n";
4789             }
4790         };
4791         if (my $err = $@) {
4792             &$add_error($opt, $err) if $err ne "skip\n";
4793         } else {
4794             # save new config if hotplug was successful
4795             delete $conf->{$opt};
4796             vmconfig_undelete_pending_option($conf, $opt);
4797             PVE::QemuConfig->write_config($vmid, $conf);
4798             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4799         }
4800     }
4801
4802     my $apply_pending_cloudinit;
4803     $apply_pending_cloudinit = sub {
4804         my ($key, $value) = @_;
4805         $apply_pending_cloudinit = sub {}; # once is enough
4806
4807         my @cloudinit_opts = keys %$confdesc_cloudinit;
4808         foreach my $opt (keys %{$conf->{pending}}) {
4809             next if !grep { $_ eq $opt } @cloudinit_opts;
4810             $conf->{$opt} = delete $conf->{pending}->{$opt};
4811         }
4812
4813         my $new_conf = { %$conf };
4814         $new_conf->{$key} = $value;
4815         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4816     };
4817
4818     foreach my $opt (keys %{$conf->{pending}}) {
4819         next if $selection && !$selection->{$opt};
4820         my $value = $conf->{pending}->{$opt};
4821         eval {
4822             if ($opt eq 'hotplug') {
4823                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4824             } elsif ($opt eq 'tablet') {
4825                 die "skip\n" if !$hotplug_features->{usb};
4826                 if ($value == 1) {
4827                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4828                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4829                         if $arch eq 'aarch64';
4830                 } elsif ($value == 0) {
4831                     vm_deviceunplug($vmid, $conf, 'tablet');
4832                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4833                 }
4834             } elsif ($opt =~ m/^usb\d+$/) {
4835                 die "skip\n";
4836                 # since we cannot reliably hot unplug usb devices
4837                 # we are disabling it
4838                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4839                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4840                 die "skip\n" if !$d;
4841                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4842             } elsif ($opt eq 'vcpus') {
4843                 die "skip\n" if !$hotplug_features->{cpu};
4844                 qemu_cpu_hotplug($vmid, $conf, $value);
4845             } elsif ($opt eq 'balloon') {
4846                 # enable/disable balloning device is not hotpluggable
4847                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4848                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4849                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4850
4851                 # allow manual ballooning if shares is set to zero
4852                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4853                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4854                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4855                 }
4856             } elsif ($opt =~ m/^net(\d+)$/) {
4857                 # some changes can be done without hotplug
4858                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4859                                     $vmid, $opt, $value, $arch, $machine_type);
4860             } elsif (is_valid_drivename($opt)) {
4861                 # some changes can be done without hotplug
4862                 my $drive = parse_drive($opt, $value);
4863                 if (drive_is_cloudinit($drive)) {
4864                     &$apply_pending_cloudinit($opt, $value);
4865                 }
4866                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4867                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4868             } elsif ($opt =~ m/^memory$/) { #dimms
4869                 die "skip\n" if !$hotplug_features->{memory};
4870                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4871             } elsif ($opt eq 'cpuunits') {
4872                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4873             } elsif ($opt eq 'cpulimit') {
4874                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4875                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4876             } else {
4877                 die "skip\n";  # skip non-hot-pluggable options
4878             }
4879         };
4880         if (my $err = $@) {
4881             &$add_error($opt, $err) if $err ne "skip\n";
4882         } else {
4883             # save new config if hotplug was successful
4884             $conf->{$opt} = $value;
4885             delete $conf->{pending}->{$opt};
4886             PVE::QemuConfig->write_config($vmid, $conf);
4887             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4888         }
4889     }
4890 }
4891
4892 sub try_deallocate_drive {
4893     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4894
4895     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4896         my $volid = $drive->{file};
4897         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4898             my $sid = PVE::Storage::parse_volume_id($volid);
4899             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4900
4901             # check if the disk is really unused
4902             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4903                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4904             PVE::Storage::vdisk_free($storecfg, $volid);
4905             return 1;
4906         } else {
4907             # If vm is not owner of this disk remove from config
4908             return 1;
4909         }
4910     }
4911
4912     return undef;
4913 }
4914
4915 sub vmconfig_delete_or_detach_drive {
4916     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4917
4918     my $drive = parse_drive($opt, $conf->{$opt});
4919
4920     my $rpcenv = PVE::RPCEnvironment::get();
4921     my $authuser = $rpcenv->get_user();
4922
4923     if ($force) {
4924         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4925         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4926     } else {
4927         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4928     }
4929 }
4930
4931 sub vmconfig_apply_pending {
4932     my ($vmid, $conf, $storecfg) = @_;
4933
4934     # cold plug
4935
4936     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4937     while (my ($opt, $force) = each %$pending_delete_hash) {
4938         die "internal error" if $opt =~ m/^unused/;
4939         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4940         if (!defined($conf->{$opt})) {
4941             vmconfig_undelete_pending_option($conf, $opt);
4942             PVE::QemuConfig->write_config($vmid, $conf);
4943         } elsif (is_valid_drivename($opt)) {
4944             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4945             vmconfig_undelete_pending_option($conf, $opt);
4946             delete $conf->{$opt};
4947             PVE::QemuConfig->write_config($vmid, $conf);
4948         } else {
4949             vmconfig_undelete_pending_option($conf, $opt);
4950             delete $conf->{$opt};
4951             PVE::QemuConfig->write_config($vmid, $conf);
4952         }
4953     }
4954
4955     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4956
4957     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4958         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4959
4960         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4961             # skip if nothing changed
4962         } elsif (is_valid_drivename($opt)) {
4963             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4964                 if defined($conf->{$opt});
4965             $conf->{$opt} = $conf->{pending}->{$opt};
4966         } else {
4967             $conf->{$opt} = $conf->{pending}->{$opt};
4968         }
4969
4970         delete $conf->{pending}->{$opt};
4971         PVE::QemuConfig->write_config($vmid, $conf);
4972     }
4973 }
4974
4975 my $safe_num_ne = sub {
4976     my ($a, $b) = @_;
4977
4978     return 0 if !defined($a) && !defined($b);
4979     return 1 if !defined($a);
4980     return 1 if !defined($b);
4981
4982     return $a != $b;
4983 };
4984
4985 my $safe_string_ne = sub {
4986     my ($a, $b) = @_;
4987
4988     return 0 if !defined($a) && !defined($b);
4989     return 1 if !defined($a);
4990     return 1 if !defined($b);
4991
4992     return $a ne $b;
4993 };
4994
4995 sub vmconfig_update_net {
4996     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4997
4998     my $newnet = parse_net($value);
4999
5000     if ($conf->{$opt}) {
5001         my $oldnet = parse_net($conf->{$opt});
5002
5003         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5004             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5005             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5006             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5007
5008             # for non online change, we try to hot-unplug
5009             die "skip\n" if !$hotplug;
5010             vm_deviceunplug($vmid, $conf, $opt);
5011         } else {
5012
5013             die "internal error" if $opt !~ m/net(\d+)/;
5014             my $iface = "tap${vmid}i$1";
5015
5016             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5017                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5018                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5019                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5020                 PVE::Network::tap_unplug($iface);
5021                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5022             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5023                 # Rate can be applied on its own but any change above needs to
5024                 # include the rate in tap_plug since OVS resets everything.
5025                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5026             }
5027
5028             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5029                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5030             }
5031
5032             return 1;
5033         }
5034     }
5035
5036     if ($hotplug) {
5037         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5038     } else {
5039         die "skip\n";
5040     }
5041 }
5042
5043 sub vmconfig_update_disk {
5044     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5045
5046     # fixme: do we need force?
5047
5048     my $drive = parse_drive($opt, $value);
5049
5050     if ($conf->{$opt}) {
5051
5052         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5053
5054             my $media = $drive->{media} || 'disk';
5055             my $oldmedia = $old_drive->{media} || 'disk';
5056             die "unable to change media type\n" if $media ne $oldmedia;
5057
5058             if (!drive_is_cdrom($old_drive)) {
5059
5060                 if ($drive->{file} ne $old_drive->{file}) {
5061
5062                     die "skip\n" if !$hotplug;
5063
5064                     # unplug and register as unused
5065                     vm_deviceunplug($vmid, $conf, $opt);
5066                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5067
5068                 } else {
5069                     # update existing disk
5070
5071                     # skip non hotpluggable value
5072                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5073                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5074                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5075                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5076                         die "skip\n";
5077                     }
5078
5079                     # apply throttle
5080                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5081                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5082                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5083                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5084                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5085                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5086                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5087                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5088                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5089                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5090                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5091                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5092                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5093                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5094                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5095                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5096                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5097                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5098
5099                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5100                                                    ($drive->{mbps} || 0)*1024*1024,
5101                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5102                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5103                                                    $drive->{iops} || 0,
5104                                                    $drive->{iops_rd} || 0,
5105                                                    $drive->{iops_wr} || 0,
5106                                                    ($drive->{mbps_max} || 0)*1024*1024,
5107                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5108                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5109                                                    $drive->{iops_max} || 0,
5110                                                    $drive->{iops_rd_max} || 0,
5111                                                    $drive->{iops_wr_max} || 0,
5112                                                    $drive->{bps_max_length} || 1,
5113                                                    $drive->{bps_rd_max_length} || 1,
5114                                                    $drive->{bps_wr_max_length} || 1,
5115                                                    $drive->{iops_max_length} || 1,
5116                                                    $drive->{iops_rd_max_length} || 1,
5117                                                    $drive->{iops_wr_max_length} || 1);
5118
5119                     }
5120
5121                     return 1;
5122                 }
5123
5124             } else { # cdrom
5125
5126                 if ($drive->{file} eq 'none') {
5127                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5128                     if (drive_is_cloudinit($old_drive)) {
5129                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5130                     }
5131                 } else {
5132                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5133                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5134                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5135                 }
5136
5137                 return 1;
5138             }
5139         }
5140     }
5141
5142     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5143     # hotplug new disks
5144     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5145     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5146 }
5147
5148 sub vm_start {
5149     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5150         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5151
5152     PVE::QemuConfig->lock_config($vmid, sub {
5153         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5154
5155         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5156
5157         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5158
5159         PVE::QemuConfig->check_lock($conf)
5160             if !($skiplock || $is_suspended);
5161
5162         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5163
5164         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5165             vmconfig_apply_pending($vmid, $conf, $storecfg);
5166             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5167         }
5168
5169         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5170
5171         my $defaults = load_defaults();
5172
5173         # set environment variable useful inside network script
5174         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5175
5176         my $local_volumes = {};
5177
5178         if ($targetstorage) {
5179             foreach_drive($conf, sub {
5180                 my ($ds, $drive) = @_;
5181
5182                 return if drive_is_cdrom($drive);
5183
5184                 my $volid = $drive->{file};
5185
5186                 return if !$volid;
5187
5188                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5189
5190                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5191                 return if $scfg->{shared};
5192                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5193             });
5194
5195             my $format = undef;
5196
5197             foreach my $opt (sort keys %$local_volumes) {
5198
5199                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5200                 my $drive = parse_drive($opt, $conf->{$opt});
5201
5202                 #if remote storage is specified, use default format
5203                 if ($targetstorage && $targetstorage ne "1") {
5204                     $storeid = $targetstorage;
5205                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5206                     $format = $defFormat;
5207                 } else {
5208                     #else we use same format than original
5209                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5210                     $format = qemu_img_format($scfg, $volid);
5211                 }
5212
5213                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5214                 my $newdrive = $drive;
5215                 $newdrive->{format} = $format;
5216                 $newdrive->{file} = $newvolid;
5217                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5218                 $local_volumes->{$opt} = $drivestr;
5219                 #pass drive to conf for command line
5220                 $conf->{$opt} = $drivestr;
5221             }
5222         }
5223
5224         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5225
5226         if ($is_suspended) {
5227             # enforce machine type on suspended vm to ensure HW compatibility
5228             $forcemachine = $conf->{runningmachine};
5229             print "Resuming suspended VM\n";
5230         }
5231
5232         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5233
5234         my $migrate_port = 0;
5235         my $migrate_uri;
5236         if ($statefile) {
5237             if ($statefile eq 'tcp') {
5238                 my $localip = "localhost";
5239                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5240                 my $nodename = PVE::INotify::nodename();
5241
5242                 if (!defined($migration_type)) {
5243                     if (defined($datacenterconf->{migration}->{type})) {
5244                         $migration_type = $datacenterconf->{migration}->{type};
5245                     } else {
5246                         $migration_type = 'secure';
5247                     }
5248                 }
5249
5250                 if ($migration_type eq 'insecure') {
5251                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5252                     if ($migrate_network_addr) {
5253                         $localip = $migrate_network_addr;
5254                     } else {
5255                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5256                     }
5257
5258                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5259                 }
5260
5261                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5262                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5263                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5264                 push @$cmd, '-incoming', $migrate_uri;
5265                 push @$cmd, '-S';
5266
5267             } elsif ($statefile eq 'unix') {
5268                 # should be default for secure migrations as a ssh TCP forward
5269                 # tunnel is not deterministic reliable ready and fails regurarly
5270                 # to set up in time, so use UNIX socket forwards
5271                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5272                 unlink $socket_addr;
5273
5274                 $migrate_uri = "unix:$socket_addr";
5275
5276                 push @$cmd, '-incoming', $migrate_uri;
5277                 push @$cmd, '-S';
5278
5279             } else {
5280                 push @$cmd, '-loadstate', $statefile;
5281             }
5282         } elsif ($paused) {
5283             push @$cmd, '-S';
5284         }
5285
5286         # host pci devices
5287         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5288           my $d = parse_hostpci($conf->{"hostpci$i"});
5289           next if !$d;
5290           my $pcidevices = $d->{pciid};
5291           foreach my $pcidevice (@$pcidevices) {
5292                 my $pciid = $pcidevice->{id};
5293
5294                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5295                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5296                 die "no pci device info for device '$pciid'\n" if !$info;
5297
5298                 if ($d->{mdev}) {
5299                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5300                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5301                 } else {
5302                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5303                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5304                     die "can't reset pci device '$pciid'\n"
5305                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5306                 }
5307           }
5308         }
5309
5310         PVE::Storage::activate_volumes($storecfg, $vollist);
5311
5312         if (-d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
5313             eval {
5314                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5315                     outfunc => sub {}, errfunc => sub {});
5316             };
5317         }
5318
5319         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5320                                                   : $defaults->{cpuunits};
5321
5322         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5323         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5324
5325         my %properties = (
5326             Slice => 'qemu.slice',
5327             KillMode => 'none',
5328             CPUShares => $cpuunits
5329         );
5330
5331         if (my $cpulimit = $conf->{cpulimit}) {
5332             $properties{CPUQuota} = int($cpulimit * 100);
5333         }
5334         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5335
5336         my $run_qemu = sub {
5337             PVE::Tools::run_fork sub {
5338                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5339                 run_command($cmd, %run_params);
5340             };
5341         };
5342
5343         if ($conf->{hugepages}) {
5344
5345             my $code = sub {
5346                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5347                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5348
5349                 PVE::QemuServer::Memory::hugepages_mount();
5350                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5351
5352                 eval { $run_qemu->() };
5353                 if (my $err = $@) {
5354                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5355                     die $err;
5356                 }
5357
5358                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5359             };
5360             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5361
5362         } else {
5363             eval { $run_qemu->() };
5364         }
5365
5366         if (my $err = $@) {
5367             # deactivate volumes if start fails
5368             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5369             die "start failed: $err";
5370         }
5371
5372         print "migration listens on $migrate_uri\n" if $migrate_uri;
5373
5374         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5375             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5376             warn $@ if $@;
5377         }
5378
5379         #start nbd server for storage migration
5380         if ($targetstorage) {
5381             my $nodename = PVE::INotify::nodename();
5382             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5383             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5384             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5385             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5386
5387             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5388
5389             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5390
5391             foreach my $opt (sort keys %$local_volumes) {
5392                 my $volid = $local_volumes->{$opt};
5393                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5394                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5395                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5396             }
5397         }
5398
5399         if ($migratedfrom) {
5400             eval {
5401                 set_migration_caps($vmid);
5402             };
5403             warn $@ if $@;
5404
5405             if ($spice_port) {
5406                 print "spice listens on port $spice_port\n";
5407                 if ($spice_ticket) {
5408                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5409                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5410                 }
5411             }
5412
5413         } else {
5414             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5415                 if !$statefile && $conf->{balloon};
5416
5417             foreach my $opt (keys %$conf) {
5418                 next if $opt !~  m/^net\d+$/;
5419                 my $nicconf = parse_net($conf->{$opt});
5420                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5421             }
5422         }
5423
5424         vm_mon_cmd_nocheck($vmid, 'qom-set',
5425                     path => "machine/peripheral/balloon0",
5426                     property => "guest-stats-polling-interval",
5427                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5428
5429         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5430             print "Resumed VM, removing state\n";
5431             delete $conf->@{qw(lock vmstate runningmachine)};
5432             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5433             PVE::Storage::vdisk_free($storecfg, $vmstate);
5434             PVE::QemuConfig->write_config($vmid, $conf);
5435         }
5436
5437         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5438     });
5439 }
5440
5441 sub vm_mon_cmd {
5442     my ($vmid, $execute, %params) = @_;
5443
5444     my $cmd = { execute => $execute, arguments => \%params };
5445     vm_qmp_command($vmid, $cmd);
5446 }
5447
5448 sub vm_mon_cmd_nocheck {
5449     my ($vmid, $execute, %params) = @_;
5450
5451     my $cmd = { execute => $execute, arguments => \%params };
5452     vm_qmp_command($vmid, $cmd, 1);
5453 }
5454
5455 sub vm_qmp_command {
5456     my ($vmid, $cmd, $nocheck) = @_;
5457
5458     my $res;
5459
5460     my $timeout;
5461     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5462         $timeout = $cmd->{arguments}->{timeout};
5463         delete $cmd->{arguments}->{timeout};
5464     }
5465
5466     eval {
5467         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5468         my $sname = qmp_socket($vmid);
5469         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5470             my $qmpclient = PVE::QMPClient->new();
5471
5472             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5473         } else {
5474             die "unable to open monitor socket\n";
5475         }
5476     };
5477     if (my $err = $@) {
5478         syslog("err", "VM $vmid qmp command failed - $err");
5479         die $err;
5480     }
5481
5482     return $res;
5483 }
5484
5485 sub vm_human_monitor_command {
5486     my ($vmid, $cmdline) = @_;
5487
5488     my $res;
5489
5490     my $cmd = {
5491         execute => 'human-monitor-command',
5492         arguments => { 'command-line' => $cmdline},
5493     };
5494
5495     return vm_qmp_command($vmid, $cmd);
5496 }
5497
5498 sub vm_commandline {
5499     my ($storecfg, $vmid, $snapname) = @_;
5500
5501     my $conf = PVE::QemuConfig->load_config($vmid);
5502
5503     if ($snapname) {
5504         my $snapshot = $conf->{snapshots}->{$snapname};
5505         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5506
5507         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5508
5509         $conf = $snapshot;
5510     }
5511
5512     my $defaults = load_defaults();
5513
5514     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5515
5516     return PVE::Tools::cmd2string($cmd);
5517 }
5518
5519 sub vm_reset {
5520     my ($vmid, $skiplock) = @_;
5521
5522     PVE::QemuConfig->lock_config($vmid, sub {
5523
5524         my $conf = PVE::QemuConfig->load_config($vmid);
5525
5526         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5527
5528         vm_mon_cmd($vmid, "system_reset");
5529     });
5530 }
5531
5532 sub get_vm_volumes {
5533     my ($conf) = @_;
5534
5535     my $vollist = [];
5536     foreach_volid($conf, sub {
5537         my ($volid, $attr) = @_;
5538
5539         return if $volid =~ m|^/|;
5540
5541         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5542         return if !$sid;
5543
5544         push @$vollist, $volid;
5545     });
5546
5547     return $vollist;
5548 }
5549
5550 sub vm_stop_cleanup {
5551     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5552
5553     eval {
5554
5555         if (!$keepActive) {
5556             my $vollist = get_vm_volumes($conf);
5557             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5558         }
5559
5560         foreach my $ext (qw(mon qmp pid vnc qga)) {
5561             unlink "/var/run/qemu-server/${vmid}.$ext";
5562         }
5563
5564         if ($conf->{ivshmem}) {
5565             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5566             # just delete it for now, VMs which have this already open do not
5567             # are affected, but new VMs will get a separated one. If this
5568             # becomes an issue we either add some sort of ref-counting or just
5569             # add a "don't delete on stop" flag to the ivshmem format.
5570             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5571         }
5572
5573         foreach my $key (keys %$conf) {
5574             next if $key !~ m/^hostpci(\d+)$/;
5575             my $hostpciindex = $1;
5576             my $d = parse_hostpci($conf->{$key});
5577             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5578
5579             foreach my $pci (@{$d->{pciid}}) {
5580                 my $pciid = $pci->{id};
5581                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5582             }
5583         }
5584
5585         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5586     };
5587     warn $@ if $@; # avoid errors - just warn
5588 }
5589
5590 # Note: use $nockeck to skip tests if VM configuration file exists.
5591 # We need that when migration VMs to other nodes (files already moved)
5592 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5593 sub vm_stop {
5594     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5595
5596     $force = 1 if !defined($force) && !$shutdown;
5597
5598     if ($migratedfrom){
5599         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5600         kill 15, $pid if $pid;
5601         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5602         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5603         return;
5604     }
5605
5606     PVE::QemuConfig->lock_config($vmid, sub {
5607
5608         my $pid = check_running($vmid, $nocheck);
5609         return if !$pid;
5610
5611         my $conf;
5612         if (!$nocheck) {
5613             $conf = PVE::QemuConfig->load_config($vmid);
5614             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5615             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5616                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5617                 $timeout = $opts->{down} if $opts->{down};
5618             }
5619             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5620         }
5621
5622         $timeout = 60 if !defined($timeout);
5623
5624         eval {
5625             if ($shutdown) {
5626                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5627                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5628                 } else {
5629                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5630                 }
5631             } else {
5632                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5633             }
5634         };
5635         my $err = $@;
5636
5637         if (!$err) {
5638             my $count = 0;
5639             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5640                 $count++;
5641                 sleep 1;
5642             }
5643
5644             if ($count >= $timeout) {
5645                 if ($force) {
5646                     warn "VM still running - terminating now with SIGTERM\n";
5647                     kill 15, $pid;
5648                 } else {
5649                     die "VM quit/powerdown failed - got timeout\n";
5650                 }
5651             } else {
5652                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5653                 return;
5654             }
5655         } else {
5656             if ($force) {
5657                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5658                 kill 15, $pid;
5659             } else {
5660                 die "VM quit/powerdown failed\n";
5661             }
5662         }
5663
5664         # wait again
5665         $timeout = 10;
5666
5667         my $count = 0;
5668         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5669             $count++;
5670             sleep 1;
5671         }
5672
5673         if ($count >= $timeout) {
5674             warn "VM still running - terminating now with SIGKILL\n";
5675             kill 9, $pid;
5676             sleep 1;
5677         }
5678
5679         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5680    });
5681 }
5682
5683 sub vm_suspend {
5684     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5685
5686     my $conf;
5687     my $path;
5688     my $storecfg;
5689     my $vmstate;
5690
5691     PVE::QemuConfig->lock_config($vmid, sub {
5692
5693         $conf = PVE::QemuConfig->load_config($vmid);
5694
5695         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5696         PVE::QemuConfig->check_lock($conf)
5697             if !($skiplock || $is_backing_up);
5698
5699         die "cannot suspend to disk during backup\n"
5700             if $is_backing_up && $includestate;
5701
5702         if ($includestate) {
5703             $conf->{lock} = 'suspending';
5704             my $date = strftime("%Y-%m-%d", localtime(time()));
5705             $storecfg = PVE::Storage::config();
5706             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5707             $path = PVE::Storage::path($storecfg, $vmstate);
5708             PVE::QemuConfig->write_config($vmid, $conf);
5709         } else {
5710             vm_mon_cmd($vmid, "stop");
5711         }
5712     });
5713
5714     if ($includestate) {
5715         # save vm state
5716         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5717
5718         eval {
5719             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5720             for(;;) {
5721                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5722                 if (!$state->{status}) {
5723                     die "savevm not active\n";
5724                 } elsif ($state->{status} eq 'active') {
5725                     sleep(1);
5726                     next;
5727                 } elsif ($state->{status} eq 'completed') {
5728                     print "State saved, quitting\n";
5729                     last;
5730                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5731                     die "query-savevm failed with error '$state->{error}'\n"
5732                 } else {
5733                     die "query-savevm returned status '$state->{status}'\n";
5734                 }
5735             }
5736         };
5737         my $err = $@;
5738
5739         PVE::QemuConfig->lock_config($vmid, sub {
5740             $conf = PVE::QemuConfig->load_config($vmid);
5741             if ($err) {
5742                 # cleanup, but leave suspending lock, to indicate something went wrong
5743                 eval {
5744                     vm_mon_cmd($vmid, "savevm-end");
5745                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5746                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5747                     delete $conf->@{qw(vmstate runningmachine)};
5748                     PVE::QemuConfig->write_config($vmid, $conf);
5749                 };
5750                 warn $@ if $@;
5751                 die $err;
5752             }
5753
5754             die "lock changed unexpectedly\n"
5755                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5756
5757             vm_qmp_command($vmid, { execute => "quit" });
5758             $conf->{lock} = 'suspended';
5759             PVE::QemuConfig->write_config($vmid, $conf);
5760         });
5761     }
5762 }
5763
5764 sub vm_resume {
5765     my ($vmid, $skiplock, $nocheck) = @_;
5766
5767     PVE::QemuConfig->lock_config($vmid, sub {
5768
5769         my $res = vm_mon_cmd($vmid, 'query-status');
5770         my $resume_cmd = 'cont';
5771
5772         if ($res->{status} && $res->{status} eq 'suspended') {
5773             $resume_cmd = 'system_wakeup';
5774         }
5775
5776         if (!$nocheck) {
5777
5778             my $conf = PVE::QemuConfig->load_config($vmid);
5779
5780             PVE::QemuConfig->check_lock($conf)
5781                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5782
5783             vm_mon_cmd($vmid, $resume_cmd);
5784
5785         } else {
5786             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5787         }
5788     });
5789 }
5790
5791 sub vm_sendkey {
5792     my ($vmid, $skiplock, $key) = @_;
5793
5794     PVE::QemuConfig->lock_config($vmid, sub {
5795
5796         my $conf = PVE::QemuConfig->load_config($vmid);
5797
5798         # there is no qmp command, so we use the human monitor command
5799         vm_human_monitor_command($vmid, "sendkey $key");
5800     });
5801 }
5802
5803 sub vm_destroy {
5804     my ($storecfg, $vmid, $skiplock) = @_;
5805
5806     PVE::QemuConfig->lock_config($vmid, sub {
5807
5808         my $conf = PVE::QemuConfig->load_config($vmid);
5809
5810         if (!check_running($vmid)) {
5811             destroy_vm($storecfg, $vmid, undef, $skiplock);
5812         } else {
5813             die "VM $vmid is running - destroy failed\n";
5814         }
5815     });
5816 }
5817
5818 # vzdump restore implementaion
5819
5820 sub tar_archive_read_firstfile {
5821     my $archive = shift;
5822
5823     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5824
5825     # try to detect archive type first
5826     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5827         die "unable to open file '$archive'\n";
5828     my $firstfile = <$fh>;
5829     kill 15, $pid;
5830     close $fh;
5831
5832     die "ERROR: archive contaions no data\n" if !$firstfile;
5833     chomp $firstfile;
5834
5835     return $firstfile;
5836 }
5837
5838 sub tar_restore_cleanup {
5839     my ($storecfg, $statfile) = @_;
5840
5841     print STDERR "starting cleanup\n";
5842
5843     if (my $fd = IO::File->new($statfile, "r")) {
5844         while (defined(my $line = <$fd>)) {
5845             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5846                 my $volid = $2;
5847                 eval {
5848                     if ($volid =~ m|^/|) {
5849                         unlink $volid || die 'unlink failed\n';
5850                     } else {
5851                         PVE::Storage::vdisk_free($storecfg, $volid);
5852                     }
5853                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5854                 };
5855                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5856             } else {
5857                 print STDERR "unable to parse line in statfile - $line";
5858             }
5859         }
5860         $fd->close();
5861     }
5862 }
5863
5864 sub restore_archive {
5865     my ($archive, $vmid, $user, $opts) = @_;
5866
5867     my $format = $opts->{format};
5868     my $comp;
5869
5870     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5871         $format = 'tar' if !$format;
5872         $comp = 'gzip';
5873     } elsif ($archive =~ m/\.tar$/) {
5874         $format = 'tar' if !$format;
5875     } elsif ($archive =~ m/.tar.lzo$/) {
5876         $format = 'tar' if !$format;
5877         $comp = 'lzop';
5878     } elsif ($archive =~ m/\.vma$/) {
5879         $format = 'vma' if !$format;
5880     } elsif ($archive =~ m/\.vma\.gz$/) {
5881         $format = 'vma' if !$format;
5882         $comp = 'gzip';
5883     } elsif ($archive =~ m/\.vma\.lzo$/) {
5884         $format = 'vma' if !$format;
5885         $comp = 'lzop';
5886     } else {
5887         $format = 'vma' if !$format; # default
5888     }
5889
5890     # try to detect archive format
5891     if ($format eq 'tar') {
5892         return restore_tar_archive($archive, $vmid, $user, $opts);
5893     } else {
5894         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5895     }
5896 }
5897
5898 sub restore_update_config_line {
5899     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5900
5901     return if $line =~ m/^\#qmdump\#/;
5902     return if $line =~ m/^\#vzdump\#/;
5903     return if $line =~ m/^lock:/;
5904     return if $line =~ m/^unused\d+:/;
5905     return if $line =~ m/^parent:/;
5906     return if $line =~ m/^template:/; # restored VM is never a template
5907
5908     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5909     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5910         # try to convert old 1.X settings
5911         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5912         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5913             my ($model, $macaddr) = split(/\=/, $devconfig);
5914             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5915             my $net = {
5916                 model => $model,
5917                 bridge => "vmbr$ind",
5918                 macaddr => $macaddr,
5919             };
5920             my $netstr = print_net($net);
5921
5922             print $outfd "net$cookie->{netcount}: $netstr\n";
5923             $cookie->{netcount}++;
5924         }
5925     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5926         my ($id, $netstr) = ($1, $2);
5927         my $net = parse_net($netstr);
5928         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5929         $netstr = print_net($net);
5930         print $outfd "$id: $netstr\n";
5931     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5932         my $virtdev = $1;
5933         my $value = $3;
5934         my $di = parse_drive($virtdev, $value);
5935         if (defined($di->{backup}) && !$di->{backup}) {
5936             print $outfd "#$line";
5937         } elsif ($map->{$virtdev}) {
5938             delete $di->{format}; # format can change on restore
5939             $di->{file} = $map->{$virtdev};
5940             $value = print_drive($vmid, $di);
5941             print $outfd "$virtdev: $value\n";
5942         } else {
5943             print $outfd $line;
5944         }
5945     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5946         my $vmgenid = $1;
5947         if ($vmgenid ne '0') {
5948             # always generate a new vmgenid if there was a valid one setup
5949             $vmgenid = generate_uuid();
5950         }
5951         print $outfd "vmgenid: $vmgenid\n";
5952     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5953         my ($uuid, $uuid_str);
5954         UUID::generate($uuid);
5955         UUID::unparse($uuid, $uuid_str);
5956         my $smbios1 = parse_smbios1($2);
5957         $smbios1->{uuid} = $uuid_str;
5958         print $outfd $1.print_smbios1($smbios1)."\n";
5959     } else {
5960         print $outfd $line;
5961     }
5962 }
5963
5964 sub scan_volids {
5965     my ($cfg, $vmid) = @_;
5966
5967     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5968
5969     my $volid_hash = {};
5970     foreach my $storeid (keys %$info) {
5971         foreach my $item (@{$info->{$storeid}}) {
5972             next if !($item->{volid} && $item->{size});
5973             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5974             $volid_hash->{$item->{volid}} = $item;
5975         }
5976     }
5977
5978     return $volid_hash;
5979 }
5980
5981 sub is_volume_in_use {
5982     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5983
5984     my $path = PVE::Storage::path($storecfg, $volid);
5985
5986     my $scan_config = sub {
5987         my ($cref, $snapname) = @_;
5988
5989         foreach my $key (keys %$cref) {
5990             my $value = $cref->{$key};
5991             if (is_valid_drivename($key)) {
5992                 next if $skip_drive && $key eq $skip_drive;
5993                 my $drive = parse_drive($key, $value);
5994                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5995                 return 1 if $volid eq $drive->{file};
5996                 if ($drive->{file} =~ m!^/!) {
5997                     return 1 if $drive->{file} eq $path;
5998                 } else {
5999                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6000                     next if !$storeid;
6001                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6002                     next if !$scfg;
6003                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6004                 }
6005             }
6006         }
6007
6008         return 0;
6009     };
6010
6011     return 1 if &$scan_config($conf);
6012
6013     undef $skip_drive;
6014
6015     foreach my $snapname (keys %{$conf->{snapshots}}) {
6016         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6017     }
6018
6019     return 0;
6020 }
6021
6022 sub update_disksize {
6023     my ($vmid, $conf, $volid_hash) = @_;
6024
6025     my $changes;
6026     my $prefix = "VM $vmid:";
6027
6028     # used and unused disks
6029     my $referenced = {};
6030
6031     # Note: it is allowed to define multiple storages with same path (alias), so
6032     # we need to check both 'volid' and real 'path' (two different volid can point
6033     # to the same path).
6034
6035     my $referencedpath = {};
6036
6037     # update size info
6038     foreach my $opt (keys %$conf) {
6039         if (is_valid_drivename($opt)) {
6040             my $drive = parse_drive($opt, $conf->{$opt});
6041             my $volid = $drive->{file};
6042             next if !$volid;
6043
6044             $referenced->{$volid} = 1;
6045             if ($volid_hash->{$volid} &&
6046                 (my $path = $volid_hash->{$volid}->{path})) {
6047                 $referencedpath->{$path} = 1;
6048             }
6049
6050             next if drive_is_cdrom($drive);
6051             next if !$volid_hash->{$volid};
6052
6053             $drive->{size} = $volid_hash->{$volid}->{size};
6054             my $new = print_drive($vmid, $drive);
6055             if ($new ne $conf->{$opt}) {
6056                 $changes = 1;
6057                 $conf->{$opt} = $new;
6058                 print "$prefix update disk '$opt' information.\n";
6059             }
6060         }
6061     }
6062
6063     # remove 'unusedX' entry if volume is used
6064     foreach my $opt (keys %$conf) {
6065         next if $opt !~ m/^unused\d+$/;
6066         my $volid = $conf->{$opt};
6067         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6068         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6069             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6070             $changes = 1;
6071             delete $conf->{$opt};
6072         }
6073
6074         $referenced->{$volid} = 1;
6075         $referencedpath->{$path} = 1 if $path;
6076     }
6077
6078     foreach my $volid (sort keys %$volid_hash) {
6079         next if $volid =~ m/vm-$vmid-state-/;
6080         next if $referenced->{$volid};
6081         my $path = $volid_hash->{$volid}->{path};
6082         next if !$path; # just to be sure
6083         next if $referencedpath->{$path};
6084         $changes = 1;
6085         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6086         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6087         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6088     }
6089
6090     return $changes;
6091 }
6092
6093 sub rescan {
6094     my ($vmid, $nolock, $dryrun) = @_;
6095
6096     my $cfg = PVE::Storage::config();
6097
6098     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6099     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6100     foreach my $stor (keys %{$cfg->{ids}}) {
6101         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6102     }
6103
6104     print "rescan volumes...\n";
6105     my $volid_hash = scan_volids($cfg, $vmid);
6106
6107     my $updatefn =  sub {
6108         my ($vmid) = @_;
6109
6110         my $conf = PVE::QemuConfig->load_config($vmid);
6111
6112         PVE::QemuConfig->check_lock($conf);
6113
6114         my $vm_volids = {};
6115         foreach my $volid (keys %$volid_hash) {
6116             my $info = $volid_hash->{$volid};
6117             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6118         }
6119
6120         my $changes = update_disksize($vmid, $conf, $vm_volids);
6121
6122         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6123     };
6124
6125     if (defined($vmid)) {
6126         if ($nolock) {
6127             &$updatefn($vmid);
6128         } else {
6129             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6130         }
6131     } else {
6132         my $vmlist = config_list();
6133         foreach my $vmid (keys %$vmlist) {
6134             if ($nolock) {
6135                 &$updatefn($vmid);
6136             } else {
6137                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6138             }
6139         }
6140     }
6141 }
6142
6143 sub restore_vma_archive {
6144     my ($archive, $vmid, $user, $opts, $comp) = @_;
6145
6146     my $readfrom = $archive;
6147
6148     my $cfg = PVE::Storage::config();
6149     my $commands = [];
6150     my $bwlimit = $opts->{bwlimit};
6151
6152     my $dbg_cmdstring = '';
6153     my $add_pipe = sub {
6154         my ($cmd) = @_;
6155         push @$commands, $cmd;
6156         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6157         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6158         $readfrom = '-';
6159     };
6160
6161     my $input = undef;
6162     if ($archive eq '-') {
6163         $input = '<&STDIN';
6164     } else {
6165         # If we use a backup from a PVE defined storage we also consider that
6166         # storage's rate limit:
6167         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6168         if (defined($volid)) {
6169             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6170             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6171             if ($readlimit) {
6172                 print STDERR "applying read rate limit: $readlimit\n";
6173                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6174                 $add_pipe->($cstream);
6175             }
6176         }
6177     }
6178
6179     if ($comp) {
6180         my $cmd;
6181         if ($comp eq 'gzip') {
6182             $cmd = ['zcat', $readfrom];
6183         } elsif ($comp eq 'lzop') {
6184             $cmd = ['lzop', '-d', '-c', $readfrom];
6185         } else {
6186             die "unknown compression method '$comp'\n";
6187         }
6188         $add_pipe->($cmd);
6189     }
6190
6191     my $tmpdir = "/var/tmp/vzdumptmp$$";
6192     rmtree $tmpdir;
6193
6194     # disable interrupts (always do cleanups)
6195     local $SIG{INT} =
6196         local $SIG{TERM} =
6197         local $SIG{QUIT} =
6198         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6199
6200     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6201     POSIX::mkfifo($mapfifo, 0600);
6202     my $fifofh;
6203
6204     my $openfifo = sub {
6205         open($fifofh, '>', $mapfifo) || die $!;
6206     };
6207
6208     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6209
6210     my $oldtimeout;
6211     my $timeout = 5;
6212
6213     my $devinfo = {};
6214
6215     my $rpcenv = PVE::RPCEnvironment::get();
6216
6217     my $conffile = PVE::QemuConfig->config_file($vmid);
6218     my $tmpfn = "$conffile.$$.tmp";
6219
6220     # Note: $oldconf is undef if VM does not exists
6221     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6222     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6223
6224     my %storage_limits;
6225
6226     my $print_devmap = sub {
6227         my $virtdev_hash = {};
6228
6229         my $cfgfn = "$tmpdir/qemu-server.conf";
6230
6231         # we can read the config - that is already extracted
6232         my $fh = IO::File->new($cfgfn, "r") ||
6233             "unable to read qemu-server.conf - $!\n";
6234
6235         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6236         if (-f $fwcfgfn) {
6237             my $pve_firewall_dir = '/etc/pve/firewall';
6238             mkdir $pve_firewall_dir; # make sure the dir exists
6239             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6240         }
6241
6242         while (defined(my $line = <$fh>)) {
6243             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6244                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6245                 die "archive does not contain data for drive '$virtdev'\n"
6246                     if !$devinfo->{$devname};
6247                 if (defined($opts->{storage})) {
6248                     $storeid = $opts->{storage} || 'local';
6249                 } elsif (!$storeid) {
6250                     $storeid = 'local';
6251                 }
6252                 $format = 'raw' if !$format;
6253                 $devinfo->{$devname}->{devname} = $devname;
6254                 $devinfo->{$devname}->{virtdev} = $virtdev;
6255                 $devinfo->{$devname}->{format} = $format;
6256                 $devinfo->{$devname}->{storeid} = $storeid;
6257
6258                 # check permission on storage
6259                 my $pool = $opts->{pool}; # todo: do we need that?
6260                 if ($user ne 'root@pam') {
6261                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6262                 }
6263
6264                 $storage_limits{$storeid} = $bwlimit;
6265
6266                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6267             }
6268         }
6269
6270         foreach my $key (keys %storage_limits) {
6271             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6272             next if !$limit;
6273             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6274             $storage_limits{$key} = $limit * 1024;
6275         }
6276
6277         foreach my $devname (keys %$devinfo) {
6278             die "found no device mapping information for device '$devname'\n"
6279                 if !$devinfo->{$devname}->{virtdev};
6280         }
6281
6282         # create empty/temp config
6283         if ($oldconf) {
6284             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6285             foreach_drive($oldconf, sub {
6286                 my ($ds, $drive) = @_;
6287
6288                 return if drive_is_cdrom($drive);
6289
6290                 my $volid = $drive->{file};
6291
6292                 return if !$volid || $volid =~ m|^/|;
6293
6294                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6295                 return if !$path || !$owner || ($owner != $vmid);
6296
6297                 # Note: only delete disk we want to restore
6298                 # other volumes will become unused
6299                 if ($virtdev_hash->{$ds}) {
6300                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6301                     if (my $err = $@) {
6302                         warn $err;
6303                     }
6304                 }
6305             });
6306
6307             # delete vmstate files
6308             # since after the restore we have no snapshots anymore
6309             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6310                 my $snap = $oldconf->{snapshots}->{$snapname};
6311                 if ($snap->{vmstate}) {
6312                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6313                     if (my $err = $@) {
6314                         warn $err;
6315                     }
6316                 }
6317             }
6318         }
6319
6320         my $map = {};
6321         foreach my $virtdev (sort keys %$virtdev_hash) {
6322             my $d = $virtdev_hash->{$virtdev};
6323             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6324             my $storeid = $d->{storeid};
6325             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6326
6327             my $map_opts = '';
6328             if (my $limit = $storage_limits{$storeid}) {
6329                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6330             }
6331
6332             # test if requested format is supported
6333             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6334             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6335             $d->{format} = $defFormat if !$supported;
6336
6337             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6338                                                   $d->{format}, undef, $alloc_size);
6339             print STDERR "new volume ID is '$volid'\n";
6340             $d->{volid} = $volid;
6341             my $path = PVE::Storage::path($cfg, $volid);
6342
6343             PVE::Storage::activate_volumes($cfg,[$volid]);
6344
6345             my $write_zeros = 1;
6346             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6347                 $write_zeros = 0;
6348             }
6349
6350             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6351
6352             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6353             $map->{$virtdev} = $volid;
6354         }
6355
6356         $fh->seek(0, 0) || die "seek failed - $!\n";
6357
6358         my $outfd = new IO::File ($tmpfn, "w") ||
6359             die "unable to write config for VM $vmid\n";
6360
6361         my $cookie = { netcount => 0 };
6362         while (defined(my $line = <$fh>)) {
6363             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6364         }
6365
6366         $fh->close();
6367         $outfd->close();
6368     };
6369
6370     eval {
6371         # enable interrupts
6372         local $SIG{INT} =
6373             local $SIG{TERM} =
6374             local $SIG{QUIT} =
6375             local $SIG{HUP} =
6376             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6377         local $SIG{ALRM} = sub { die "got timeout\n"; };
6378
6379         $oldtimeout = alarm($timeout);
6380
6381         my $parser = sub {
6382             my $line = shift;
6383
6384             print "$line\n";
6385
6386             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6387                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6388                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6389             } elsif ($line =~ m/^CTIME: /) {
6390                 # we correctly received the vma config, so we can disable
6391                 # the timeout now for disk allocation (set to 10 minutes, so
6392                 # that we always timeout if something goes wrong)
6393                 alarm(600);
6394                 &$print_devmap();
6395                 print $fifofh "done\n";
6396                 my $tmp = $oldtimeout || 0;
6397                 $oldtimeout = undef;
6398                 alarm($tmp);
6399                 close($fifofh);
6400             }
6401         };
6402
6403         print "restore vma archive: $dbg_cmdstring\n";
6404         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6405     };
6406     my $err = $@;
6407
6408     alarm($oldtimeout) if $oldtimeout;
6409
6410     my $vollist = [];
6411     foreach my $devname (keys %$devinfo) {
6412         my $volid = $devinfo->{$devname}->{volid};
6413         push @$vollist, $volid if $volid;
6414     }
6415
6416     PVE::Storage::deactivate_volumes($cfg, $vollist);
6417
6418     unlink $mapfifo;
6419
6420     if ($err) {
6421         rmtree $tmpdir;
6422         unlink $tmpfn;
6423
6424         foreach my $devname (keys %$devinfo) {
6425             my $volid = $devinfo->{$devname}->{volid};
6426             next if !$volid;
6427             eval {
6428                 if ($volid =~ m|^/|) {
6429                     unlink $volid || die 'unlink failed\n';
6430                 } else {
6431                     PVE::Storage::vdisk_free($cfg, $volid);
6432                 }
6433                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6434             };
6435             print STDERR "unable to cleanup '$volid' - $@" if $@;
6436         }
6437         die $err;
6438     }
6439
6440     rmtree $tmpdir;
6441
6442     rename($tmpfn, $conffile) ||
6443         die "unable to commit configuration file '$conffile'\n";
6444
6445     PVE::Cluster::cfs_update(); # make sure we read new file
6446
6447     eval { rescan($vmid, 1); };
6448     warn $@ if $@;
6449 }
6450
6451 sub restore_tar_archive {
6452     my ($archive, $vmid, $user, $opts) = @_;
6453
6454     if ($archive ne '-') {
6455         my $firstfile = tar_archive_read_firstfile($archive);
6456         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6457             if $firstfile ne 'qemu-server.conf';
6458     }
6459
6460     my $storecfg = PVE::Storage::config();
6461
6462     # destroy existing data - keep empty config
6463     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6464     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6465
6466     my $tocmd = "/usr/lib/qemu-server/qmextract";
6467
6468     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6469     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6470     $tocmd .= ' --prealloc' if $opts->{prealloc};
6471     $tocmd .= ' --info' if $opts->{info};
6472
6473     # tar option "xf" does not autodetect compression when read from STDIN,
6474     # so we pipe to zcat
6475     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6476         PVE::Tools::shellquote("--to-command=$tocmd");
6477
6478     my $tmpdir = "/var/tmp/vzdumptmp$$";
6479     mkpath $tmpdir;
6480
6481     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6482     local $ENV{VZDUMP_VMID} = $vmid;
6483     local $ENV{VZDUMP_USER} = $user;
6484
6485     my $conffile = PVE::QemuConfig->config_file($vmid);
6486     my $tmpfn = "$conffile.$$.tmp";
6487
6488     # disable interrupts (always do cleanups)
6489     local $SIG{INT} =
6490         local $SIG{TERM} =
6491         local $SIG{QUIT} =
6492         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6493
6494     eval {
6495         # enable interrupts
6496         local $SIG{INT} =
6497             local $SIG{TERM} =
6498             local $SIG{QUIT} =
6499             local $SIG{HUP} =
6500             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6501
6502         if ($archive eq '-') {
6503             print "extracting archive from STDIN\n";
6504             run_command($cmd, input => "<&STDIN");
6505         } else {
6506             print "extracting archive '$archive'\n";
6507             run_command($cmd);
6508         }
6509
6510         return if $opts->{info};
6511
6512         # read new mapping
6513         my $map = {};
6514         my $statfile = "$tmpdir/qmrestore.stat";
6515         if (my $fd = IO::File->new($statfile, "r")) {
6516             while (defined (my $line = <$fd>)) {
6517                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6518                     $map->{$1} = $2 if $1;
6519                 } else {
6520                     print STDERR "unable to parse line in statfile - $line\n";
6521                 }
6522             }
6523             $fd->close();
6524         }
6525
6526         my $confsrc = "$tmpdir/qemu-server.conf";
6527
6528         my $srcfd = new IO::File($confsrc, "r") ||
6529             die "unable to open file '$confsrc'\n";
6530
6531         my $outfd = new IO::File ($tmpfn, "w") ||
6532             die "unable to write config for VM $vmid\n";
6533
6534         my $cookie = { netcount => 0 };
6535         while (defined (my $line = <$srcfd>)) {
6536             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6537         }
6538
6539         $srcfd->close();
6540         $outfd->close();
6541     };
6542     my $err = $@;
6543
6544     if ($err) {
6545
6546         unlink $tmpfn;
6547
6548         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6549
6550         die $err;
6551     }
6552
6553     rmtree $tmpdir;
6554
6555     rename $tmpfn, $conffile ||
6556         die "unable to commit configuration file '$conffile'\n";
6557
6558     PVE::Cluster::cfs_update(); # make sure we read new file
6559
6560     eval { rescan($vmid, 1); };
6561     warn $@ if $@;
6562 };
6563
6564 sub foreach_storage_used_by_vm {
6565     my ($conf, $func) = @_;
6566
6567     my $sidhash = {};
6568
6569     foreach_drive($conf, sub {
6570         my ($ds, $drive) = @_;
6571         return if drive_is_cdrom($drive);
6572
6573         my $volid = $drive->{file};
6574
6575         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6576         $sidhash->{$sid} = $sid if $sid;
6577     });
6578
6579     foreach my $sid (sort keys %$sidhash) {
6580         &$func($sid);
6581     }
6582 }
6583
6584 sub do_snapshots_with_qemu {
6585     my ($storecfg, $volid) = @_;
6586
6587     my $storage_name = PVE::Storage::parse_volume_id($volid);
6588
6589     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6590         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6591         return 1;
6592     }
6593
6594     if ($volid =~ m/\.(qcow2|qed)$/){
6595         return 1;
6596     }
6597
6598     return undef;
6599 }
6600
6601 sub qga_check_running {
6602     my ($vmid, $nowarn) = @_;
6603
6604     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6605     if ($@) {
6606         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6607         return 0;
6608     }
6609     return 1;
6610 }
6611
6612 sub template_create {
6613     my ($vmid, $conf, $disk) = @_;
6614
6615     my $storecfg = PVE::Storage::config();
6616
6617     foreach_drive($conf, sub {
6618         my ($ds, $drive) = @_;
6619
6620         return if drive_is_cdrom($drive);
6621         return if $disk && $ds ne $disk;
6622
6623         my $volid = $drive->{file};
6624         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6625
6626         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6627         $drive->{file} = $voliddst;
6628         $conf->{$ds} = print_drive($vmid, $drive);
6629         PVE::QemuConfig->write_config($vmid, $conf);
6630     });
6631 }
6632
6633 sub convert_iscsi_path {
6634     my ($path) = @_;
6635
6636     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6637         my $portal = $1;
6638         my $target = $2;
6639         my $lun = $3;
6640
6641         my $initiator_name = get_initiator_name();
6642
6643         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6644                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6645     }
6646
6647     die "cannot convert iscsi path '$path', unkown format\n";
6648 }
6649
6650 sub qemu_img_convert {
6651     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6652
6653     my $storecfg = PVE::Storage::config();
6654     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6655     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6656
6657     if ($src_storeid && $dst_storeid) {
6658
6659         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6660
6661         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6662         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6663
6664         my $src_format = qemu_img_format($src_scfg, $src_volname);
6665         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6666
6667         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6668         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6669
6670         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6671         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6672
6673         my $cmd = [];
6674         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6675         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6676         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6677         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6678
6679         if ($src_is_iscsi) {
6680             push @$cmd, '--image-opts';
6681             $src_path = convert_iscsi_path($src_path);
6682         } else {
6683             push @$cmd, '-f', $src_format;
6684         }
6685
6686         if ($dst_is_iscsi) {
6687             push @$cmd, '--target-image-opts';
6688             $dst_path = convert_iscsi_path($dst_path);
6689         } else {
6690             push @$cmd, '-O', $dst_format;
6691         }
6692
6693         push @$cmd, $src_path;
6694
6695         if (!$dst_is_iscsi && $is_zero_initialized) {
6696             push @$cmd, "zeroinit:$dst_path";
6697         } else {
6698             push @$cmd, $dst_path;
6699         }
6700
6701         my $parser = sub {
6702             my $line = shift;
6703             if($line =~ m/\((\S+)\/100\%\)/){
6704                 my $percent = $1;
6705                 my $transferred = int($size * $percent / 100);
6706                 my $remaining = $size - $transferred;
6707
6708                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6709             }
6710
6711         };
6712
6713         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6714         my $err = $@;
6715         die "copy failed: $err" if $err;
6716     }
6717 }
6718
6719 sub qemu_img_format {
6720     my ($scfg, $volname) = @_;
6721
6722     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6723         return $1;
6724     } else {
6725         return "raw";
6726     }
6727 }
6728
6729 sub qemu_drive_mirror {
6730     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6731
6732     $jobs = {} if !$jobs;
6733
6734     my $qemu_target;
6735     my $format;
6736     $jobs->{"drive-$drive"} = {};
6737
6738     if ($dst_volid =~ /^nbd:/) {
6739         $qemu_target = $dst_volid;
6740         $format = "nbd";
6741     } else {
6742         my $storecfg = PVE::Storage::config();
6743         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6744
6745         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6746
6747         $format = qemu_img_format($dst_scfg, $dst_volname);
6748
6749         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6750
6751         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6752     }
6753
6754     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6755     $opts->{format} = $format if $format;
6756
6757     print "drive mirror is starting for drive-$drive\n";
6758
6759     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6760
6761     if (my $err = $@) {
6762         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6763         die "mirroring error: $err";
6764     }
6765
6766     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6767 }
6768
6769 sub qemu_drive_mirror_monitor {
6770     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6771
6772     eval {
6773         my $err_complete = 0;
6774
6775         while (1) {
6776             die "storage migration timed out\n" if $err_complete > 300;
6777
6778             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6779
6780             my $running_mirror_jobs = {};
6781             foreach my $stat (@$stats) {
6782                 next if $stat->{type} ne 'mirror';
6783                 $running_mirror_jobs->{$stat->{device}} = $stat;
6784             }
6785
6786             my $readycounter = 0;
6787
6788             foreach my $job (keys %$jobs) {
6789
6790                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6791                     print "$job : finished\n";
6792                     delete $jobs->{$job};
6793                     next;
6794                 }
6795
6796                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6797
6798                 my $busy = $running_mirror_jobs->{$job}->{busy};
6799                 my $ready = $running_mirror_jobs->{$job}->{ready};
6800                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6801                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6802                     my $remaining = $total - $transferred;
6803                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6804
6805                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6806                 }
6807
6808                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6809             }
6810
6811             last if scalar(keys %$jobs) == 0;
6812
6813             if ($readycounter == scalar(keys %$jobs)) {
6814                 print "all mirroring jobs are ready \n";
6815                 last if $skipcomplete; #do the complete later
6816
6817                 if ($vmiddst && $vmiddst != $vmid) {
6818                     my $agent_running = $qga && qga_check_running($vmid);
6819                     if ($agent_running) {
6820                         print "freeze filesystem\n";
6821                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6822                     } else {
6823                         print "suspend vm\n";
6824                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6825                     }
6826
6827                     # if we clone a disk for a new target vm, we don't switch the disk
6828                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6829
6830                     if ($agent_running) {
6831                         print "unfreeze filesystem\n";
6832                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6833                     } else {
6834                         print "resume vm\n";
6835                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6836                     }
6837
6838                     last;
6839                 } else {
6840
6841                     foreach my $job (keys %$jobs) {
6842                         # try to switch the disk if source and destination are on the same guest
6843                         print "$job: Completing block job...\n";
6844
6845                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6846                         if ($@ =~ m/cannot be completed/) {
6847                             print "$job: Block job cannot be completed, try again.\n";
6848                             $err_complete++;
6849                         }else {
6850                             print "$job: Completed successfully.\n";
6851                             $jobs->{$job}->{complete} = 1;
6852                         }
6853                     }
6854                 }
6855             }
6856             sleep 1;
6857         }
6858     };
6859     my $err = $@;
6860
6861     if ($err) {
6862         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6863         die "mirroring error: $err";
6864     }
6865
6866 }
6867
6868 sub qemu_blockjobs_cancel {
6869     my ($vmid, $jobs) = @_;
6870
6871     foreach my $job (keys %$jobs) {
6872         print "$job: Cancelling block job\n";
6873         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6874         $jobs->{$job}->{cancel} = 1;
6875     }
6876
6877     while (1) {
6878         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6879
6880         my $running_jobs = {};
6881         foreach my $stat (@$stats) {
6882             $running_jobs->{$stat->{device}} = $stat;
6883         }
6884
6885         foreach my $job (keys %$jobs) {
6886
6887             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6888                 print "$job: Done.\n";
6889                 delete $jobs->{$job};
6890             }
6891         }
6892
6893         last if scalar(keys %$jobs) == 0;
6894
6895         sleep 1;
6896     }
6897 }
6898
6899 sub clone_disk {
6900     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6901         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6902
6903     my $newvolid;
6904
6905     if (!$full) {
6906         print "create linked clone of drive $drivename ($drive->{file})\n";
6907         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6908         push @$newvollist, $newvolid;
6909     } else {
6910
6911         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6912         $storeid = $storage if $storage;
6913
6914         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6915         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6916
6917         print "create full clone of drive $drivename ($drive->{file})\n";
6918         my $name = undef;
6919         if (drive_is_cloudinit($drive)) {
6920             $name = "vm-$newvmid-cloudinit";
6921             $snapname = undef;
6922             # cloudinit only supports raw and qcow2 atm:
6923             if ($dst_format eq 'qcow2') {
6924                 $name .= '.qcow2';
6925             } elsif ($dst_format ne 'raw') {
6926                 die "clone: unhandled format for cloudinit image\n";
6927             }
6928         }
6929         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6930         push @$newvollist, $newvolid;
6931
6932         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6933
6934         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6935         if (!$running || $snapname) {
6936             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6937         } else {
6938
6939             my $kvmver = get_running_qemu_version ($vmid);
6940             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6941                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6942                     if $drive->{iothread};
6943             }
6944
6945             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6946         }
6947     }
6948
6949     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6950
6951     my $disk = $drive;
6952     $disk->{format} = undef;
6953     $disk->{file} = $newvolid;
6954     $disk->{size} = $size;
6955
6956     return $disk;
6957 }
6958
6959 # this only works if VM is running
6960 sub get_current_qemu_machine {
6961     my ($vmid) = @_;
6962
6963     my $cmd = { execute => 'query-machines', arguments => {} };
6964     my $res = vm_qmp_command($vmid, $cmd);
6965
6966     my ($current, $default);
6967     foreach my $e (@$res) {
6968         $default = $e->{name} if $e->{'is-default'};
6969         $current = $e->{name} if $e->{'is-current'};
6970     }
6971
6972     # fallback to the default machine if current is not supported by qemu
6973     return $current || $default || 'pc';
6974 }
6975
6976 sub get_running_qemu_version {
6977     my ($vmid) = @_;
6978     my $cmd = { execute => 'query-version', arguments => {} };
6979     my $res = vm_qmp_command($vmid, $cmd);
6980     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6981 }
6982
6983 sub qemu_machine_feature_enabled {
6984     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6985
6986     my $current_major;
6987     my $current_minor;
6988
6989     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6990
6991         $current_major = $3;
6992         $current_minor = $4;
6993
6994     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6995
6996         $current_major = $1;
6997         $current_minor = $2;
6998     }
6999
7000     return 1 if $current_major > $version_major ||
7001                 ($current_major == $version_major &&
7002                  $current_minor >= $version_minor);
7003 }
7004
7005 sub qemu_machine_pxe {
7006     my ($vmid, $conf, $machine) = @_;
7007
7008     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
7009
7010     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7011         $machine .= '.pxe';
7012     }
7013
7014     return $machine;
7015 }
7016
7017 sub qemu_use_old_bios_files {
7018     my ($machine_type) = @_;
7019
7020     return if !$machine_type;
7021
7022     my $use_old_bios_files = undef;
7023
7024     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7025         $machine_type = $1;
7026         $use_old_bios_files = 1;
7027     } else {
7028         my $kvmver = kvm_user_version();
7029         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7030         # load new efi bios files on migration. So this hack is required to allow
7031         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7032         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7033         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7034     }
7035
7036     return ($use_old_bios_files, $machine_type);
7037 }
7038
7039 sub create_efidisk($$$$$) {
7040     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7041
7042     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7043     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7044
7045     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7046     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7047     PVE::Storage::activate_volumes($storecfg, [$volid]);
7048
7049     my $path = PVE::Storage::path($storecfg, $volid);
7050     eval {
7051         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7052     };
7053     die "Copying EFI vars image failed: $@" if $@;
7054
7055     return ($volid, $vars_size);
7056 }
7057
7058 sub vm_iothreads_list {
7059     my ($vmid) = @_;
7060
7061     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7062
7063     my $iothreads = {};
7064     foreach my $iothread (@$res) {
7065         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7066     }
7067
7068     return $iothreads;
7069 }
7070
7071 sub scsihw_infos {
7072     my ($conf, $drive) = @_;
7073
7074     my $maxdev = 0;
7075
7076     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7077         $maxdev = 7;
7078     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7079         $maxdev = 1;
7080     } else {
7081         $maxdev = 256;
7082     }
7083
7084     my $controller = int($drive->{index} / $maxdev);
7085     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7086
7087     return ($maxdev, $controller, $controller_prefix);
7088 }
7089
7090 sub add_hyperv_enlightenments {
7091     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7092
7093     return if $winversion < 6;
7094     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7095
7096     if ($gpu_passthrough || defined($hv_vendor_id)) {
7097         $hv_vendor_id //= 'proxmox';
7098         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7099     }
7100
7101     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7102         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7103         push @$cpuFlags , 'hv_vapic';
7104         push @$cpuFlags , 'hv_time';
7105     } else {
7106         push @$cpuFlags , 'hv_spinlocks=0xffff';
7107     }
7108
7109     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7110         push @$cpuFlags , 'hv_reset';
7111         push @$cpuFlags , 'hv_vpindex';
7112         push @$cpuFlags , 'hv_runtime';
7113     }
7114
7115     if ($winversion >= 7) {
7116         push @$cpuFlags , 'hv_relaxed';
7117
7118         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7119             push @$cpuFlags , 'hv_synic';
7120             push @$cpuFlags , 'hv_stimer';
7121         }
7122     }
7123 }
7124
7125 sub windows_version {
7126     my ($ostype) = @_;
7127
7128     return 0 if !$ostype;
7129
7130     my $winversion = 0;
7131
7132     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7133         $winversion = 5;
7134     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7135         $winversion = 6;
7136     } elsif ($ostype =~ m/^win(\d+)$/) {
7137         $winversion = $1;
7138     }
7139
7140     return $winversion;
7141 }
7142
7143 sub resolve_dst_disk_format {
7144         my ($storecfg, $storeid, $src_volname, $format) = @_;
7145         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7146
7147         if (!$format) {
7148             # if no target format is specified, use the source disk format as hint
7149             if ($src_volname) {
7150                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7151                 $format = qemu_img_format($scfg, $src_volname);
7152             } else {
7153                 return $defFormat;
7154             }
7155         }
7156
7157         # test if requested format is supported - else use default
7158         my $supported = grep { $_ eq $format } @$validFormats;
7159         $format = $defFormat if !$supported;
7160         return $format;
7161 }
7162
7163 sub resolve_first_disk {
7164     my $conf = shift;
7165     my @disks = PVE::QemuServer::valid_drive_names();
7166     my $firstdisk;
7167     foreach my $ds (reverse @disks) {
7168         next if !$conf->{$ds};
7169         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7170         next if PVE::QemuServer::drive_is_cdrom($disk);
7171         $firstdisk = $ds;
7172     }
7173     return $firstdisk;
7174 }
7175
7176 sub generate_uuid {
7177     my ($uuid, $uuid_str);
7178     UUID::generate($uuid);
7179     UUID::unparse($uuid, $uuid_str);
7180     return $uuid_str;
7181 }
7182
7183 sub generate_smbios1_uuid {
7184     return "uuid=".generate_uuid();
7185 }
7186
7187 sub nbd_stop {
7188     my ($vmid) = @_;
7189
7190     vm_mon_cmd($vmid, 'nbd-server-stop');
7191 }
7192
7193 # bash completion helper
7194
7195 sub complete_backup_archives {
7196     my ($cmdname, $pname, $cvalue) = @_;
7197
7198     my $cfg = PVE::Storage::config();
7199
7200     my $storeid;
7201
7202     if ($cvalue =~ m/^([^:]+):/) {
7203         $storeid = $1;
7204     }
7205
7206     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7207
7208     my $res = [];
7209     foreach my $id (keys %$data) {
7210         foreach my $item (@{$data->{$id}}) {
7211             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7212             push @$res, $item->{volid} if defined($item->{volid});
7213         }
7214     }
7215
7216     return $res;
7217 }
7218
7219 my $complete_vmid_full = sub {
7220     my ($running) = @_;
7221
7222     my $idlist = vmstatus();
7223
7224     my $res = [];
7225
7226     foreach my $id (keys %$idlist) {
7227         my $d = $idlist->{$id};
7228         if (defined($running)) {
7229             next if $d->{template};
7230             next if $running && $d->{status} ne 'running';
7231             next if !$running && $d->{status} eq 'running';
7232         }
7233         push @$res, $id;
7234
7235     }
7236     return $res;
7237 };
7238
7239 sub complete_vmid {
7240     return &$complete_vmid_full();
7241 }
7242
7243 sub complete_vmid_stopped {
7244     return &$complete_vmid_full(0);
7245 }
7246
7247 sub complete_vmid_running {
7248     return &$complete_vmid_full(1);
7249 }
7250
7251 sub complete_storage {
7252
7253     my $cfg = PVE::Storage::config();
7254     my $ids = $cfg->{ids};
7255
7256     my $res = [];
7257     foreach my $sid (keys %$ids) {
7258         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7259         next if !$ids->{$sid}->{content}->{images};
7260         push @$res, $sid;
7261     }
7262
7263     return $res;
7264 }
7265
7266 1;